Skip to content

Commit f3e07ff

Browse files
thymikeethymikee
authored
test: interaction contract suite with registry-driven coverage gate (ADR 0011 Layer 3) (#1092)
* fix: preserve the runner's non-hittable fallback marker through direct selector press The direct iOS selector press handler spread successText('Tapped <selector>') after the runner payload, clobbering the 'tapped via non-hittable coordinate fallback' message that directIosSelectorFallbackDetails keys on — so maestroNonHittableCoordinateFallbackUsed could never be true end-to-end (the existing unit test passes because it mocks dispatchCommand above this layer). Found by the ADR 0011 Layer-3 maestro-fallback contract scenario. Share the marker string as MAESTRO_NON_HITTABLE_FALLBACK_MESSAGE and keep it as the success message when the runner reports fallback usage. * test: interaction contract suite with registry-driven coverage gate (ADR 0011 Layer 3) test/integration/interaction-contract/ holds one scenario file per dispatch path, each with a sibling .coverage.ts manifest declaring which guarantee matrix cells it proves (scenario strings double as the vitest test titles). index.ts aggregates the manifests statically, and a new Layer-3 gate in src/contracts/__tests__/interaction-contract-coverage.test.ts fails when any enforced (runtime/runner/delegated) cell lacks a scenario or when a scenario claims a waived/inapplicable cell — coverage of the matrix is by construction in both directions. Path forcing is natural (no test-only env switch needed): selector/ref targets take the runtime path, a tapTarget backend takes the native-ref fast path, x/y takes the coordinate path, and simple-selector clicks on an iOS provider transcript take the direct runner path (the transcript itself proves which path ran via assertComplete). Fixtures are the permanent Bluesky shapes: closed drawer, drawer + visible twin, edge-grazing container, covered button, non-hittable cell. Refs #1081
1 parent 03b666b commit f3e07ff

22 files changed

Lines changed: 1365 additions & 12 deletions

‎src/contracts/__tests__/interaction-contract-coverage.test.ts‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
import { test } from 'vitest';
2+
import assert from 'node:assert/strict';
3+
import {
4+
INTERACTION_DISPATCH_PATHS,
5+
INTERACTION_GUARANTEES,
6+
INTERACTION_PATH_IDS,
7+
} from '../interaction-guarantees.ts';
8+
import { CONTRACT_COVERAGE } from '../../../test/integration/interaction-contract/index.ts';
9+
10+
// ADR 0011 Layer-3 gate: the contract scenario suite is registry-driven. Every
11+
// matrix cell that claims enforcement (runtime, runner, or delegated) must be
12+
// proven by at least one scenario in test/integration/interaction-contract/,
13+
// and no scenario may claim a waived or inapplicable cell — coverage of the
14+
// matrix is by construction, not reviewer memory, in both directions.
15+
16+
const ENFORCED_KINDS: ReadonlySet<string> = new Set(['runtime', 'runner', 'delegated']);
17+
18+
function cellKey(pathId: string, guarantee: string): string {
19+
return `${pathId}/${guarantee}`;
20+
}
21+
22+
test('contract coverage entries reference real matrix cells and named scenarios', () => {
23+
for (const entry of CONTRACT_COVERAGE) {
24+
assert.ok(
25+
(INTERACTION_PATH_IDS as readonly string[]).includes(entry.path),
26+
`coverage entry references unknown path "${entry.path}"`,
27+
);
28+
assert.ok(
29+
(INTERACTION_GUARANTEES as readonly string[]).includes(entry.guarantee),
30+
`coverage entry references unknown guarantee "${entry.guarantee}"`,
31+
);
32+
assert.ok(
33+
entry.scenario.trim().length > 10,
34+
`${cellKey(entry.path, entry.guarantee)}: scenario name must describe what it proves`,
35+
);
36+
}
37+
});
38+
39+
test('contract coverage never claims a waived or inapplicable cell', () => {
40+
for (const entry of CONTRACT_COVERAGE) {
41+
const enforcement = INTERACTION_DISPATCH_PATHS[entry.path].guarantees[entry.guarantee];
42+
// Overclaiming is an error too: a scenario tagged onto a waived cell would
43+
// make the debt list look repaid without flipping the matrix cell.
44+
assert.ok(
45+
ENFORCED_KINDS.has(enforcement.kind),
46+
`${cellKey(entry.path, entry.guarantee)} is "${enforcement.kind}" — drop the coverage entry or flip the matrix cell in the same PR`,
47+
);
48+
}
49+
});
50+
51+
test('every enforced matrix cell has at least one contract scenario', () => {
52+
const covered = new Set(CONTRACT_COVERAGE.map((entry) => cellKey(entry.path, entry.guarantee)));
53+
const missing: string[] = [];
54+
for (const pathId of INTERACTION_PATH_IDS) {
55+
for (const guarantee of INTERACTION_GUARANTEES) {
56+
const enforcement = INTERACTION_DISPATCH_PATHS[pathId].guarantees[guarantee];
57+
if (!ENFORCED_KINDS.has(enforcement.kind)) continue;
58+
if (!covered.has(cellKey(pathId, guarantee))) missing.push(cellKey(pathId, guarantee));
59+
}
60+
}
61+
assert.deepEqual(
62+
missing,
63+
[],
64+
'enforced matrix cells without a contract scenario — add one under test/integration/interaction-contract/ and register its manifest in index.ts',
65+
);
66+
});

‎src/core/dispatch-interactions.ts‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,11 @@ import {
4949
} from './dispatch-series.ts';
5050
import type { RunnerSequenceStep } from '../platforms/apple/core/runner/runner-contract.ts';
5151
import type { DispatchContext } from './dispatch-context.ts';
52-
import type { Interactor, RunnerCallOptions } from './interactor-types.ts';
52+
import {
53+
MAESTRO_NON_HITTABLE_FALLBACK_MESSAGE,
54+
type Interactor,
55+
type RunnerCallOptions,
56+
} from './interactor-types.ts';
5357

5458
type ScrollTarget = {
5559
direction: ScrollDirection;
@@ -185,10 +189,16 @@ async function handleDirectElementSelectorPress(
185189
throw new AppError('UNSUPPORTED_OPERATION', 'direct element selector tap is not supported');
186190
}
187191
const result = await interactor.tapElementSelector(selector);
192+
// Keep the runner's non-hittable-fallback marker: the daemon reports
193+
// maestroNonHittableCoordinateFallbackUsed from this message, so replacing
194+
// it with the generic success text would silently hide fallback usage.
195+
const usedNonHittableFallback = result?.message === MAESTRO_NON_HITTABLE_FALLBACK_MESSAGE;
188196
return {
189197
selector: selector.raw,
190198
...(result ?? {}),
191-
...successText(`Tapped ${selector.raw}`),
199+
...successText(
200+
usedNonHittableFallback ? MAESTRO_NON_HITTABLE_FALLBACK_MESSAGE : `Tapped ${selector.raw}`,
201+
),
192202
};
193203
}
194204

‎src/core/interactor-types.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,14 @@ export type ElementSelectorTapOptions = {
5555
allowNonHittableCoordinateFallback?: boolean;
5656
};
5757

58+
/**
59+
* Wire marker the XCTest runner puts in `message` when a selector tap used the
60+
* Maestro non-hittable coordinate fallback (RunnerTests+CommandExecution.swift).
61+
* The direct-selector press handler must preserve it so the daemon can report
62+
* `maestroNonHittableCoordinateFallbackUsed` truthfully.
63+
*/
64+
export const MAESTRO_NON_HITTABLE_FALLBACK_MESSAGE = 'tapped via non-hittable coordinate fallback';
65+
5866
export type SnapshotOptions = BaseSnapshotOptions & {
5967
appBundleId?: string;
6068
includeRects?: boolean;

‎src/daemon/handlers/interaction-touch.ts‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ import {
4040
import { getActiveAndroidSnapshotFreshness } from '../android-snapshot-freshness.ts';
4141
import { emitDiagnostic } from '../../utils/diagnostics.ts';
4242
import { dispatchCommand, type CommandFlags } from '../../core/dispatch.ts';
43+
import { MAESTRO_NON_HITTABLE_FALLBACK_MESSAGE } from '../../core/interactor-types.ts';
4344
import {
4445
isDirectIosSelectorFallbackError,
4546
readSimpleIosSelectorTarget,
@@ -253,7 +254,15 @@ function readDirectIosSelectorTapTarget(params: {
253254
if (target.kind !== 'selector') return null;
254255
if (hasNonDefaultClickOptions(flags)) return null;
255256
if (flags?.verify === true) return null;
256-
const selector = readSimpleIosSelectorTarget({ session, selectorExpression: target.selector });
257+
return readDirectSelectorWithMaestroFallback(session, target.selector, flags);
258+
}
259+
260+
function readDirectSelectorWithMaestroFallback(
261+
session: SessionState,
262+
selectorExpression: string,
263+
flags: CommandFlags | undefined,
264+
): DirectIosSelectorTarget | null {
265+
const selector = readSimpleIosSelectorTarget({ session, selectorExpression });
257266
if (!selector) return null;
258267
return {
259268
...selector,
@@ -363,7 +372,7 @@ function directIosSelectorFallbackDetails(
363372
data: Record<string, unknown>,
364373
): Record<string, unknown> {
365374
if (!selector.allowNonHittableCoordinateFallback) return {};
366-
const used = data.message === 'tapped via non-hittable coordinate fallback';
375+
const used = data.message === MAESTRO_NON_HITTABLE_FALLBACK_MESSAGE;
367376
return {
368377
maestroNonHittableCoordinateFallbackAllowed: true,
369378
maestroNonHittableCoordinateFallbackUsed: used,
@@ -463,14 +472,7 @@ function readDirectIosSelectorFillTarget(params: {
463472
const { session, target, flags } = params;
464473
if (target.kind !== 'selector') return null;
465474
if (flags?.verify === true) return null;
466-
const selector = readSimpleIosSelectorTarget({ session, selectorExpression: target.selector });
467-
if (!selector) return null;
468-
return {
469-
...selector,
470-
...(flags?.maestro?.allowNonHittableCoordinateFallback
471-
? { allowNonHittableCoordinateFallback: true }
472-
: {}),
473-
};
475+
return readDirectSelectorWithMaestroFallback(session, target.selector, flags);
474476
}
475477

476478
async function dispatchDirectIosSelectorFill(

‎test/integration/interaction-contract/coordinate.contract.test.ts‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
import assert from 'node:assert/strict';
2+
import { test } from 'vitest';
3+
import type { InteractionGuarantee } from '../../../src/contracts/interaction-guarantees.ts';
4+
import { makeSnapshotState } from '../../../src/__tests__/test-utils/index.ts';
5+
import { assertRpcError, assertRpcOk } from '../provider-scenarios/assertions.ts';
6+
import { scenarioName } from './coverage-manifest.ts';
7+
import { COORDINATE_COVERAGE } from './coordinate.coverage.ts';
8+
import { viewportOnlySnapshot } from './fixtures.ts';
9+
import { createContractDevice } from './runtime-harness.ts';
10+
import { runnerTapEntry, runnerTapErrorEntry, withIosContractDaemon } from './daemon-harness.ts';
11+
12+
// ADR 0011 Layer 3, coordinate path: raw x/y tap, intentionally minimal
13+
// semantics. Path forcing is natural: a point target never resolves nodes.
14+
15+
const scenario = (guarantee: InteractionGuarantee): string =>
16+
scenarioName(COORDINATE_COVERAGE, guarantee);
17+
18+
test(scenario('offscreen'), async () => {
19+
const device = createContractDevice(viewportOnlySnapshot(), {
20+
tap: async () => ({ ok: true }),
21+
});
22+
23+
const result = await device.interactions.click(
24+
{ kind: 'point', x: 500, y: 500 },
25+
{ session: 'default' },
26+
);
27+
28+
// Coordinate semantics: the escape hatch forwards the tap but must warn —
29+
// never a silent out-of-viewport no-op, never a refusal.
30+
assert.equal(result.kind, 'point');
31+
assert.match(result.warning ?? '', /outside the last-known viewport \(400x800\)/);
32+
});
33+
34+
test(scenario('verifyEvidence'), async () => {
35+
let captureCount = 0;
36+
const device = createContractDevice(viewportOnlySnapshot(), {
37+
captureSnapshot: async () => {
38+
captureCount += 1;
39+
if (captureCount === 1) return { snapshot: viewportOnlySnapshot() };
40+
return { snapshot: makeSnapshotState([]) };
41+
},
42+
tap: async () => ({ ok: true }),
43+
});
44+
45+
const result = await device.interactions.click(
46+
{ kind: 'point', x: 10, y: 20 },
47+
{ session: 'default', verify: true },
48+
);
49+
50+
assert.equal(result.kind, 'point');
51+
assert.ok(result.evidence);
52+
assert.equal(result.evidence?.changedFromBefore, true);
53+
});
54+
55+
test(scenario('errorTaxonomy'), async () => {
56+
await withIosContractDaemon(
57+
[runnerTapErrorEntry(new Error('runner tap crashed'))],
58+
async (daemon) => {
59+
const press = await daemon.callCommand('press', ['100', '200']);
60+
// An unclassified backend failure gets the shared fallback classification
61+
// from normalizeError: stable code, original message, actionable hint.
62+
const error = assertRpcError(press, 'UNKNOWN', /runner tap crashed/);
63+
assert.ok(typeof error.hint === 'string' && error.hint.length > 0);
64+
assert.ok(typeof error.diagnosticId === 'string');
65+
},
66+
);
67+
});
68+
69+
test(scenario('responseConstruction'), async () => {
70+
await withIosContractDaemon([runnerTapEntry({ x: 100, y: 200 })], async (daemon) => {
71+
const press = await daemon.callCommand('press', ['100', '200']);
72+
const data = assertRpcOk(press);
73+
// Canonical point response set: the tapped coordinates, no fabricated
74+
// identity fields (the path has no resolved node by design).
75+
assert.equal(data.x, 100);
76+
assert.equal(data.y, 200);
77+
assert.equal(data.ref, undefined);
78+
assert.equal(data.selector, undefined);
79+
});
80+
});

‎test/integration/interaction-contract/coordinate.coverage.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
import { definePathCoverage } from './coverage-manifest.ts';
2+
3+
export const COORDINATE_COVERAGE = definePathCoverage('coordinate', {
4+
offscreen: 'coordinate offscreen: out-of-viewport point is forwarded with a viewport warning',
5+
responseConstruction:
6+
'coordinate responseConstruction: daemon press x y response carries the canonical point field set',
7+
verifyEvidence:
8+
'coordinate verifyEvidence: point click --verify returns a digest with change detection',
9+
errorTaxonomy: 'coordinate errorTaxonomy: backend failure surfaces as a normalized daemon error',
10+
});

‎test/integration/interaction-contract/coverage-manifest.ts‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
import type {
2+
InteractionGuarantee,
3+
InteractionPathId,
4+
} from '../../../src/contracts/interaction-guarantees.ts';
5+
6+
/**
7+
* ADR 0011 Layer 3: the machine-readable claim of which interaction guarantee
8+
* matrix cells a scenario file proves. Every `<path>.contract.test.ts` has a
9+
* sibling `<path>.coverage.ts` exporting one of these manifests (kept out of
10+
* the test file so the coverage gate can import it without re-registering the
11+
* scenarios), and `index.ts` aggregates them statically for the gate test in
12+
* `src/contracts/__tests__/interaction-contract-coverage.test.ts`.
13+
*
14+
* Scenario strings double as the vitest test titles — the test files derive
15+
* their titles from the manifest (via `scenarioName`/`scenarioNames`), so a
16+
* claim without a matching scenario cannot be written by accident.
17+
*/
18+
export type ContractCoverageEntry = {
19+
path: InteractionPathId;
20+
guarantee: InteractionGuarantee;
21+
scenario: string;
22+
};
23+
24+
export function definePathCoverage(
25+
path: InteractionPathId,
26+
scenarios: Partial<Record<InteractionGuarantee, string | readonly string[]>>,
27+
): readonly ContractCoverageEntry[] {
28+
return Object.entries(scenarios).flatMap(([guarantee, names]) =>
29+
(typeof names === 'string' ? [names] : (names ?? [])).map((scenario) => ({
30+
path,
31+
guarantee: guarantee as InteractionGuarantee,
32+
scenario,
33+
})),
34+
);
35+
}
36+
37+
export function scenarioNames(
38+
coverage: readonly ContractCoverageEntry[],
39+
guarantee: InteractionGuarantee,
40+
): string[] {
41+
const names = coverage
42+
.filter((entry) => entry.guarantee === guarantee)
43+
.map((entry) => entry.scenario);
44+
if (names.length === 0) {
45+
throw new Error(`no contract scenario declared for guarantee "${guarantee}"`);
46+
}
47+
return names;
48+
}
49+
50+
export function scenarioName(
51+
coverage: readonly ContractCoverageEntry[],
52+
guarantee: InteractionGuarantee,
53+
): string {
54+
return scenarioNames(coverage, guarantee)[0]!;
55+
}

‎test/integration/interaction-contract/daemon-harness.ts‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
import { assertRpcOk } from '../provider-scenarios/assertions.ts';
2+
import { PROVIDER_SCENARIO_IOS_SIMULATOR } from '../provider-scenarios/fixtures.ts';
3+
import {
4+
createProviderScenarioHarness,
5+
withProviderScenarioResource,
6+
type ProviderScenarioHarness,
7+
} from '../provider-scenarios/harness.ts';
8+
import {
9+
createAppleRunnerProviderFromTranscript,
10+
createRecordingAppleToolProvider,
11+
simctlListDevicesHandler,
12+
} from '../provider-scenarios/providers.ts';
13+
import {
14+
createProviderTranscript,
15+
type ProviderScenarioProviderEntry,
16+
type ProviderScenarioTranscript,
17+
} from '../provider-scenarios/transcript.ts';
18+
19+
const CONTRACT_APP = 'com.example.app';
20+
const CONTRACT_DEVICE_ID = PROVIDER_SCENARIO_IOS_SIMULATOR.id;
21+
22+
/**
23+
* Provider-transcript harness for contract scenarios whose path involves the
24+
* iOS runner (direct-ios-selector, maestro-non-hittable-fallback) or that
25+
* prove daemon-level response construction. The transcript is the proof
26+
* vehicle: `assertComplete` after `run` guarantees exactly the scripted
27+
* runner conversation happened — a path that dispatched differently either
28+
* consumes an unexpected entry or leaves one behind.
29+
*/
30+
export async function withIosContractDaemon(
31+
entries: readonly ProviderScenarioProviderEntry[],
32+
run: (daemon: ProviderScenarioHarness, transcript: ProviderScenarioTranscript) => Promise<void>,
33+
): Promise<void> {
34+
const transcript = createProviderTranscript(entries);
35+
const appleRunnerProvider = createAppleRunnerProviderFromTranscript(transcript, 'ios.runner');
36+
const appleTool = createRecordingAppleToolProvider({
37+
simctl: simctlListDevicesHandler('com.apple.CoreSimulator.SimRuntime.iOS-18-0', [
38+
{ name: PROVIDER_SCENARIO_IOS_SIMULATOR.name, udid: CONTRACT_DEVICE_ID },
39+
]),
40+
});
41+
42+
await withProviderScenarioResource(
43+
async () =>
44+
await createProviderScenarioHarness({
45+
appleRunnerProvider: () => appleRunnerProvider,
46+
appleToolProvider: () => appleTool.provider,
47+
deviceInventoryProvider: async () => [PROVIDER_SCENARIO_IOS_SIMULATOR],
48+
}),
49+
async (daemon) => {
50+
const open = await daemon.callCommand('open', [CONTRACT_APP], {
51+
platform: 'ios',
52+
udid: CONTRACT_DEVICE_ID,
53+
});
54+
assertRpcOk(open);
55+
await run(daemon, transcript);
56+
transcript.assertComplete();
57+
},
58+
);
59+
}
60+
61+
export function runnerSnapshotEntry(nodes: readonly unknown[]): ProviderScenarioProviderEntry {
62+
return {
63+
command: 'ios.runner.snapshot',
64+
deviceId: CONTRACT_DEVICE_ID,
65+
platform: 'apple',
66+
result: { nodes, truncated: false },
67+
};
68+
}
69+
70+
export function runnerTapEntry(result: Record<string, unknown>): ProviderScenarioProviderEntry {
71+
return {
72+
command: 'ios.runner.tap',
73+
deviceId: CONTRACT_DEVICE_ID,
74+
platform: 'apple',
75+
result,
76+
};
77+
}
78+
79+
export function runnerTapErrorEntry(error: Error): ProviderScenarioProviderEntry {
80+
return {
81+
command: 'ios.runner.tap',
82+
deviceId: CONTRACT_DEVICE_ID,
83+
platform: 'apple',
84+
error,
85+
};
86+
}

0 commit comments

Comments
 (0)