Skip to content

Commit b2ed38e

Browse files
committed
fix: exclude raw baselines from corroboration and prove the pin end to end (review)
Raw baselines could not be pinned: the raw diagnostic plan keeps tree-first error propagation by contract and is never rerouted by the penalty or the preferred backend, so preserving 'raw: true' on the probe recreated exactly the backend-mismatch false failure this PR removes. Corroboration now declines raw baselines up front (they are diagnostics, not evidence baselines) with a regression pinning that no probe capture is dispatched at all. The wire is now regression-proven at every hop: a dispatch-level test drives dispatchCommand with the context flag and asserts the emitted RunnerCommand carries preferredBackend (red if handleSnapshotCommand or the interactor stops forwarding); the injected-transport test asserts the interactor's snapshot payload both ways; and a runner unit test decodes the wire JSON, projects it through the extracted snapshotOptions(from:), and composes it with the plan rule — pinned regular plan defers to privateAX-first, RAW plan stays untouched. Executed on-simulator; added to the ios.yml regression list.
1 parent c09d5e3 commit b2ed38e

8 files changed

Lines changed: 152 additions & 3 deletions

File tree

‎.github/workflows/ios.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,7 @@ jobs:
107107
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testBoundedSystemModalProbeTimeoutRecoversThenReleasesOnDrainForSnapshotRaw \
108108
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testSnapshotTraversalIdentityPreservesSameOriginNodesWithDifferentBounds \
109109
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testPreferredPrivateAXBackendPlansAsPenalized \
110+
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testDecodedPreferredBackendReachesOptionsAndApplicablePlan \
110111
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testRunnerScreenshotStabilitySettledNeedsEnoughSamples \
111112
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testRunnerScreenshotStabilitySettledTrueWhenWindowMatches \
112113
-only-testing:AgentDeviceRunnerUITests/RunnerTests/testRunnerScreenshotStabilitySettledFalseOnMidWindowMismatch \

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1358,14 +1358,20 @@ extension RunnerTests {
13581358
}
13591359
}
13601360

1361-
private func executeSnapshotPrepared(command: Command, activeApp: XCUIApplication) throws -> Response {
1362-
let options = SnapshotOptions(
1361+
/// Pure command→options projection, extracted so the runner unit bundle can
1362+
/// prove the decoded wire field actually reaches capture options (#1634 P2).
1363+
static func snapshotOptions(from command: Command) -> SnapshotOptions {
1364+
SnapshotOptions(
13631365
interactiveOnly: command.interactiveOnly ?? false,
13641366
depth: command.depth,
13651367
scope: command.scope,
13661368
raw: command.raw ?? false,
13671369
preferredBackend: command.preferredBackend
13681370
)
1371+
}
1372+
1373+
private func executeSnapshotPrepared(command: Command, activeApp: XCUIApplication) throws -> Response {
1374+
let options = Self.snapshotOptions(from: command)
13691375
do {
13701376
let payload: DataPayload
13711377
if options.raw {

‎apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -685,6 +685,35 @@ extension RunnerTests {
685685
XCTAssertEqual(Self.xcTestChannelStateFirstFailure(.boundedXCTestProbe)?.code, "budget")
686686
}
687687

688+
/// #1634 P2: the decoded wire field must reach capture options and its
689+
/// applicable plan. A pinned REGULAR capture defers to privateAX-first; the
690+
/// RAW diagnostic plan is never rerouted by the pin — raw keeps tree-first
691+
/// error propagation, which is exactly why raw baselines are excluded from
692+
/// corroboration daemon-side.
693+
func testDecodedPreferredBackendReachesOptionsAndApplicablePlan() throws {
694+
let json = #"{"command":"snapshot","preferredBackend":"private-ax"}"#
695+
let command = try JSONDecoder().decode(Command.self, from: Data(json.utf8))
696+
let options = Self.snapshotOptions(from: command)
697+
XCTAssertEqual(options.preferredBackend, "private-ax")
698+
XCTAssertFalse(options.raw)
699+
700+
let treated = Self.snapshotXCTestChannelTreatedAsPenalized(
701+
penalized: false, preferredBackend: options.preferredBackend)
702+
let pinned = Self.effectiveSnapshotCapturePlan(
703+
Self.regularVisiblePlan, xCTestChannelPenalized: treated)
704+
XCTAssertEqual(pinned.plan, [.privateAX])
705+
XCTAssertEqual(pinned.xCTestChannelState, .deferredToIndependentBackend)
706+
707+
let raw = Self.effectiveSnapshotCapturePlan(
708+
Self.rawDiagnosticPlan, xCTestChannelPenalized: treated)
709+
XCTAssertEqual(raw.plan, Self.rawDiagnosticPlan)
710+
711+
// A command without the field decodes to no pin and a normal plan.
712+
let bare = try JSONDecoder().decode(
713+
Command.self, from: Data(#"{"command":"snapshot"}"#.utf8))
714+
XCTAssertNil(Self.snapshotOptions(from: bare).preferredBackend)
715+
}
716+
688717
/// Same-backend evidence probes: a daemon-pinned private-AX capture takes the
689718
/// penalized route even with a healthy channel, so tap-outcome corroboration
690719
/// baselines and probes are always captured by the same backend (backends are
Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
import { beforeEach, test, vi } from 'vitest';
2+
import assert from 'node:assert/strict';
3+
4+
vi.mock('../../platforms/apple/core/runner/runner-client.ts', async (importOriginal) => {
5+
const actual =
6+
await importOriginal<typeof import('../../platforms/apple/core/runner/runner-client.ts')>();
7+
return { ...actual, runAppleRunnerCommand: vi.fn() };
8+
});
9+
10+
import { dispatchCommand } from '../dispatch.ts';
11+
import { runAppleRunnerCommand } from '../../platforms/apple/core/runner/runner-client.ts';
12+
import { IOS_SIMULATOR } from '../../__tests__/test-utils/device-fixtures.ts';
13+
14+
const mockRunAppleRunnerCommand = vi.mocked(runAppleRunnerCommand);
15+
16+
beforeEach(() => {
17+
vi.resetAllMocks();
18+
mockRunAppleRunnerCommand.mockResolvedValue({
19+
nodes: [{ index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }],
20+
});
21+
});
22+
23+
// #1634 P2: proves the whole daemon-side wire — dispatch context ->
24+
// handleSnapshotCommand -> interactor -> emitted RunnerCommand. Removing the
25+
// forwarding at any of those hops turns this red; the daemon corroboration
26+
// test (mocked dispatch) and the Swift plan-gate test cover the layers on
27+
// either side of it.
28+
test('dispatch snapshot forwards snapshotPreferredBackend to the runner command', async () => {
29+
await dispatchCommand(IOS_SIMULATOR, 'snapshot', [], undefined, {
30+
snapshotPreferredBackend: 'private-ax',
31+
});
32+
33+
const call = mockRunAppleRunnerCommand.mock.calls.find(
34+
([, command]) => command.command === 'snapshot',
35+
);
36+
assert.ok(call, 'expected a snapshot runner command');
37+
assert.equal(call[1].preferredBackend, 'private-ax');
38+
});
39+
40+
test('dispatch snapshot without a pin emits no preferredBackend', async () => {
41+
await dispatchCommand(IOS_SIMULATOR, 'snapshot', [], undefined, {});
42+
43+
const call = mockRunAppleRunnerCommand.mock.calls.find(
44+
([, command]) => command.command === 'snapshot',
45+
);
46+
assert.ok(call, 'expected a snapshot runner command');
47+
assert.equal(call[1].preferredBackend, undefined);
48+
});

‎src/daemon/handlers/__tests__/interaction-ios-tap-outcome-fixtures.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,14 +40,15 @@ export const imageViewerNodes: RawSnapshotNode[] = [
4040
export function snapshot(
4141
nodes: RawSnapshotNode[],
4242
backend: 'tree' | 'queries' | 'private-ax' = 'tree',
43+
options: { raw?: boolean } = {},
4344
) {
4445
return buildSnapshotState(
4546
{
4647
nodes,
4748
backend: 'xctest',
4849
quality: { state: 'healthy', backend },
4950
},
50-
{ snapshotInteractiveOnly: false },
51+
{ snapshotInteractiveOnly: false, ...(options.raw ? { snapshotRaw: true } : {}) },
5152
);
5253
}
5354

‎src/daemon/handlers/__tests__/interaction-ios-tap-outcome.test.ts‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,39 @@ test('a private-ax baseline pins the corroboration probe to private-ax', async (
157157
expect(snapshotContexts[0]?.snapshotPreferredBackend).toBe('private-ax');
158158
});
159159

160+
test('a raw baseline is excluded from corroboration entirely (#1634 P1)', async () => {
161+
// The raw diagnostic plan keeps tree-first error propagation by contract and
162+
// is never rerouted by the pin, so a raw private-AX baseline could not be
163+
// matched same-backend — corroboration must decline up front (no probe
164+
// capture at all) and the recorded failure surfaces unchanged.
165+
const sessionName = 'ios-raw-baseline-no-corroboration';
166+
const sessionStore = makeSessionStore();
167+
sessionStore.set(
168+
sessionName,
169+
makeIosSession(sessionName, {
170+
appBundleId: 'com.example.app',
171+
snapshot: snapshot(profileNodes, 'private-ax', { raw: true }),
172+
}),
173+
);
174+
mockDispatch.mockImplementation(async (_device, command) => {
175+
if (command === 'press') {
176+
throw new AppError(
177+
'XCTEST_RECORDED_FAILURE',
178+
'XCTest recorded a failure while executing tap; the action may not have been performed.',
179+
);
180+
}
181+
if (command === 'snapshot') return snapshotPayload(imageViewerNodes, 'private-ax');
182+
return {};
183+
});
184+
185+
const response = await runClick(sessionStore, sessionName);
186+
187+
expect(response?.ok).toBe(false);
188+
if (response && !response.ok) expect(response.error.code).toBe('XCTEST_RECORDED_FAILURE');
189+
// Declined before any probe: no corroboration snapshot was dispatched.
190+
expect(mockDispatch.mock.calls.filter((call) => call[1] === 'snapshot')).toHaveLength(0);
191+
});
192+
160193
test('a tree baseline does not pin the corroboration probe backend', async () => {
161194
const sessionName = 'ios-tree-baseline-unpinned-corroboration';
162195
const sessionStore = makeSessionStore();

‎src/daemon/handlers/interaction-ios-tap-outcome.ts‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,20 @@ function readCorroborationBaseline(
9191
}
9292
const presentation = readSnapshotPresentation(snapshot.presentationKey);
9393
if (!presentation) return undefined;
94+
// Raw baselines are excluded from corroboration entirely: the probe would
95+
// replay `raw: true`, and the raw diagnostic plan keeps tree-first error
96+
// propagation by contract — it is never rerouted by the penalty or by a
97+
// preferred backend, so a raw private-AX baseline could not be matched
98+
// same-backend and would recreate the mismatch false failure. Raw captures
99+
// are diagnostics, not evidence baselines.
100+
if (presentation.raw) {
101+
emitDiagnostic({
102+
level: 'debug',
103+
phase: 'ios_tap_failure_corroboration_raw_baseline',
104+
data: { presentationKey: snapshot.presentationKey },
105+
});
106+
return undefined;
107+
}
94108
return { snapshot, presentation };
95109
}
96110

‎src/platforms/apple/__tests__/interactor-runner-provider.test.ts‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,23 @@ test('snapshot over the injected transport keeps the shared xctest result shape'
148148
assert.equal(result.nodes?.length, 2);
149149
});
150150

151+
// #1634 P2: the backend pin must actually reach the wire — the daemon test
152+
// stops at the dispatch context and the Swift test starts at the parsed
153+
// command, so this is the assertion that fails if the interactor stops
154+
// forwarding preferredBackend into the emitted RunnerCommand.
155+
test('snapshot forwards preferredBackend into the emitted runner command', async () => {
156+
const calls: RecordedRunnerCall[] = [];
157+
const interactor = createAppleInteractor(IOS_SIMULATOR, {}, recordingRunnerProvider(calls));
158+
159+
await interactor.snapshot({ preferredBackend: 'private-ax' });
160+
await interactor.snapshot();
161+
162+
const snapshots = calls.filter((call) => call.command.command === 'snapshot');
163+
assert.equal(snapshots.length, 2);
164+
assert.equal(snapshots[0]?.command.preferredBackend, 'private-ax');
165+
assert.equal(snapshots[1]?.command.preferredBackend, undefined);
166+
});
167+
151168
function recordingRunnerProvider(calls: RecordedRunnerCall[]): AppleRunnerProvider {
152169
return {
153170
runCommand: async (_device, command, options) => {

0 commit comments

Comments
 (0)