|
| 1 | +// Characterization of the shipped reporter contract for the MAESTRO adapter (#1478 P3). |
| 2 | +// |
| 3 | +// The sibling session-test-reporter-values.test.ts covers the same contract for native `.ad`. |
| 4 | +// Both are needed, and neither substitutes for the other: P3 replaced a request-global |
| 5 | +// AsyncLocalStorage with an explicit per-attempt `onStep` port, and that port has two |
| 6 | +// independent forwarding chains. The native chain is |
| 7 | +// scheduler sink -> runReplayScriptFile -> executeReplayActions -> onStep?.(...) |
| 8 | +// and the Maestro chain is a different set of links entirely: |
| 9 | +// scheduler sink -> runReplayScriptFile -> runTypedMaestroReplayFile |
| 10 | +// -> createMaestroReplayObserver({ onStep }) -> actionStarted -> onStep?.(...) |
| 11 | +// |
| 12 | +// An `.ad`-only reporter test stays green if any Maestro link is deleted, so `onTestStep` |
| 13 | +// would silently stop firing for every `test --maestro` run with no gate objecting. That is |
| 14 | +// the same defect class as the dropped diagnosticId/logPath (#1501) and the dropped reporter |
| 15 | +// `hint` (#1505): a ratchet that keeps passing while a shipped value disappears. |
| 16 | +// |
| 17 | +// These tests document today's behavior; they do not propose behavior. |
| 18 | + |
| 19 | +// Maestro replay resolves a target device and captures view hierarchies through |
| 20 | +// core/dispatch. These fixtures model no runner, so both are stubbed: the device resolves to a |
| 21 | +// fixed Android emulator, and `snapshot` returns an empty hierarchy (which is what makes |
| 22 | +// `assertNotVisible` pass deterministically without a screen). |
| 23 | +import { expect, test, vi } from 'vitest'; |
| 24 | + |
| 25 | +vi.mock('../../../core/dispatch.ts', async (importOriginal) => { |
| 26 | + const actual = await importOriginal<typeof import('../../../core/dispatch.ts')>(); |
| 27 | + return { |
| 28 | + ...actual, |
| 29 | + resolveTargetDevice: vi.fn(async () => ({ |
| 30 | + platform: 'android', |
| 31 | + id: 'emulator-5554', |
| 32 | + name: 'Pixel 8', |
| 33 | + kind: 'emulator', |
| 34 | + booted: true, |
| 35 | + })), |
| 36 | + dispatchCommand: vi.fn(async () => { |
| 37 | + throw new Error('no device runner available in this test'); |
| 38 | + }), |
| 39 | + dispatchGestureViewport: vi.fn(async () => ({ x: 0, y: 0, width: 400, height: 800 })), |
| 40 | + }; |
| 41 | +}); |
| 42 | + |
| 43 | +import fs from 'node:fs'; |
| 44 | +import os from 'node:os'; |
| 45 | +import path from 'node:path'; |
| 46 | +import type { ReplaySuiteResult } from '@agent-device/contracts/replay'; |
| 47 | +import { handleSessionCommands } from '../session.ts'; |
| 48 | +import { SessionStore } from '../../session-store.ts'; |
| 49 | +import type { DaemonRequest, DaemonResponse } from '../../types.ts'; |
| 50 | +import { withRequestProgressSink } from '../../../request/progress.ts'; |
| 51 | +import { |
| 52 | + getReplayTestReporterExitCode, |
| 53 | + runReplayTestReporterProgress, |
| 54 | + runReplayTestReporters, |
| 55 | +} from '../../../replay/test/reporters/registry.ts'; |
| 56 | +import type { |
| 57 | + ReplayTestReporter, |
| 58 | + ReplayTestReporterContext, |
| 59 | +} from '../../../replay/test/reporters/types.ts'; |
| 60 | + |
| 61 | +type RecordedHook = { |
| 62 | + hook: 'onSuiteStart' | 'onTestStart' | 'onTestStep' | 'onTestResult' | 'onSuiteEnd'; |
| 63 | + value: Record<string, unknown>; |
| 64 | +}; |
| 65 | + |
| 66 | +const reporterContext: ReplayTestReporterContext = { |
| 67 | + stdout: { isTTY: false, write() {} }, |
| 68 | + stderr: { isTTY: false, write() {} }, |
| 69 | +}; |
| 70 | + |
| 71 | +/** |
| 72 | + * Two steps, so `stepIndex`/`stepTotal` are distinguishable and one step carries a value while |
| 73 | + * the other does not. The flow `name` is what populates the reporter `title` — a value only the |
| 74 | + * Maestro path can produce, since `.ad` scripts have no title. |
| 75 | + */ |
| 76 | +const MAESTRO_FLOW = [ |
| 77 | + 'appId: demo.app', |
| 78 | + 'name: Login flow', |
| 79 | + '---', |
| 80 | + '- assertNotVisible: "Nope"', |
| 81 | + '- back', |
| 82 | + '', |
| 83 | +].join('\n'); |
| 84 | + |
| 85 | +function createRecordingReporter(): { reporter: ReplayTestReporter; hooks: RecordedHook[] } { |
| 86 | + const hooks: RecordedHook[] = []; |
| 87 | + return { |
| 88 | + hooks, |
| 89 | + reporter: { |
| 90 | + name: 'recording', |
| 91 | + onSuiteStart: (suite) => void hooks.push({ hook: 'onSuiteStart', value: { ...suite } }), |
| 92 | + onTestStart: (testCase) => void hooks.push({ hook: 'onTestStart', value: { ...testCase } }), |
| 93 | + onTestStep: (testCase) => void hooks.push({ hook: 'onTestStep', value: { ...testCase } }), |
| 94 | + onTestResult: (testCase) => void hooks.push({ hook: 'onTestResult', value: { ...testCase } }), |
| 95 | + onSuiteEnd: (suite) => void hooks.push({ hook: 'onSuiteEnd', value: { ...suite } }), |
| 96 | + getExitCode: (suite) => (suite.failed > 0 ? 1 : 0), |
| 97 | + }, |
| 98 | + }; |
| 99 | +} |
| 100 | + |
| 101 | +function makeSessionStore(root: string): SessionStore { |
| 102 | + return new SessionStore(path.join(root, 'sessions')); |
| 103 | +} |
| 104 | + |
| 105 | +/** |
| 106 | + * Runs `test --maestro` exactly as the daemon does, feeding every progress event through the |
| 107 | + * shipped reporter registry — the same translation and dispatch a `--reporter ./mine.mjs` |
| 108 | + * module gets. |
| 109 | + */ |
| 110 | +async function runMaestroSuiteThroughReporter(params: { |
| 111 | + root: string; |
| 112 | + requestId: string; |
| 113 | + flags?: DaemonRequest['flags']; |
| 114 | + invoke: (req: DaemonRequest) => Promise<DaemonResponse>; |
| 115 | +}): Promise<{ hooks: RecordedHook[]; suite: ReplaySuiteResult; exitCode: number }> { |
| 116 | + const { reporter, hooks } = createRecordingReporter(); |
| 117 | + const reporters = [reporter]; |
| 118 | + const response = await withRequestProgressSink( |
| 119 | + (event) => runReplayTestReporterProgress(reporters, event, reporterContext), |
| 120 | + async () => |
| 121 | + await handleSessionCommands({ |
| 122 | + req: { |
| 123 | + token: 't', |
| 124 | + session: 'default', |
| 125 | + command: 'test', |
| 126 | + positionals: [params.root], |
| 127 | + meta: { cwd: params.root, requestId: params.requestId }, |
| 128 | + flags: { replayBackend: 'maestro', platform: 'android', ...params.flags }, |
| 129 | + }, |
| 130 | + sessionName: 'default', |
| 131 | + logPath: path.join(os.tmpdir(), 'daemon.log'), |
| 132 | + sessionStore: makeSessionStore(params.root), |
| 133 | + invoke: params.invoke, |
| 134 | + }), |
| 135 | + ); |
| 136 | + |
| 137 | + expect(response?.ok, JSON.stringify(response)).toBeTruthy(); |
| 138 | + if (!response?.ok) throw new Error('Expected a successful suite response.'); |
| 139 | + const suite = response.data as unknown as ReplaySuiteResult; |
| 140 | + await runReplayTestReporters(reporters, suite, reporterContext); |
| 141 | + return { hooks, suite, exitCode: getReplayTestReporterExitCode(reporters, suite) }; |
| 142 | +} |
| 143 | + |
| 144 | +/** `snapshot` returns an empty hierarchy; every other command succeeds. */ |
| 145 | +function maestroInvoke(req: DaemonRequest): DaemonResponse { |
| 146 | + return req.command === 'snapshot' |
| 147 | + ? { ok: true, data: { createdAt: 0, nodes: [] } } |
| 148 | + : { ok: true, data: {} }; |
| 149 | +} |
| 150 | + |
| 151 | +test('a Maestro suite reaches the reporter with step payloads and attempt-scoped identity', async () => { |
| 152 | + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-reporter-maestro-')); |
| 153 | + fs.writeFileSync(path.join(root, '01-login.yaml'), MAESTRO_FLOW); |
| 154 | + |
| 155 | + const { hooks, suite, exitCode } = await runMaestroSuiteThroughReporter({ |
| 156 | + root, |
| 157 | + requestId: 'suite-maestro', |
| 158 | + invoke: async (req) => maestroInvoke(req), |
| 159 | + }); |
| 160 | + |
| 161 | + expect(hooks.map((entry) => entry.hook)).toEqual([ |
| 162 | + 'onSuiteStart', |
| 163 | + 'onTestStart', |
| 164 | + 'onTestStep', |
| 165 | + 'onTestStep', |
| 166 | + 'onTestResult', |
| 167 | + 'onSuiteEnd', |
| 168 | + ]); |
| 169 | + |
| 170 | + const suiteArtifactsDir = path.join(root, '.agent-device', 'test-artifacts', 'suite-maestro'); |
| 171 | + const testArtifactsDir = path.join(suiteArtifactsDir, '01-login.yaml'); |
| 172 | + const session = 'default:test:suite-maestro:1-01-login:attempt-1'; |
| 173 | + |
| 174 | + // The Maestro flow's `name` becomes the reporter `title`. Discovery reads it through |
| 175 | + // `inspectMaestroFlow`, so this is a shipped reporter value the native scenario cannot cover. |
| 176 | + expect(hooks[1]?.value).toMatchObject({ |
| 177 | + file: path.join(root, '01-login.yaml'), |
| 178 | + title: 'Login flow', |
| 179 | + index: 1, |
| 180 | + total: 1, |
| 181 | + attempt: undefined, |
| 182 | + maxAttempts: 1, |
| 183 | + session, |
| 184 | + artifactsDir: testArtifactsDir, |
| 185 | + }); |
| 186 | + |
| 187 | + // The two step events: payload comes from the Maestro engine through the observer's |
| 188 | + // `onStep`, while file/index/total/attempt/session/artifactsDir come from the scheduler's |
| 189 | + // per-attempt context. Before P3 the latter half came from request-global AsyncLocalStorage. |
| 190 | + const steps = hooks.filter((entry) => entry.hook === 'onTestStep').map((entry) => entry.value); |
| 191 | + expect(steps).toHaveLength(2); |
| 192 | + expect('status' in steps[0]!).toBe(false); |
| 193 | + expect(steps[0]).toMatchObject({ |
| 194 | + file: path.join(root, '01-login.yaml'), |
| 195 | + title: 'Login flow', |
| 196 | + index: 1, |
| 197 | + total: 1, |
| 198 | + attempt: 1, |
| 199 | + maxAttempts: 1, |
| 200 | + session, |
| 201 | + artifactsDir: testArtifactsDir, |
| 202 | + stepIndex: 1, |
| 203 | + stepTotal: 2, |
| 204 | + stepCommand: 'assertNotVisible', |
| 205 | + stepValue: 'Nope', |
| 206 | + }); |
| 207 | + expect(steps[1]).toMatchObject({ |
| 208 | + index: 1, |
| 209 | + total: 1, |
| 210 | + attempt: 1, |
| 211 | + session, |
| 212 | + artifactsDir: testArtifactsDir, |
| 213 | + stepIndex: 2, |
| 214 | + stepTotal: 2, |
| 215 | + stepCommand: 'back', |
| 216 | + }); |
| 217 | + // `back` carries no progress value, and the observer must not invent one. |
| 218 | + expect(steps[1]!.stepValue).toBe(undefined); |
| 219 | + |
| 220 | + expect(hooks[4]?.value).toMatchObject({ |
| 221 | + status: 'pass', |
| 222 | + title: 'Login flow', |
| 223 | + attempt: 1, |
| 224 | + maxAttempts: 1, |
| 225 | + session, |
| 226 | + artifactsDir: testArtifactsDir, |
| 227 | + }); |
| 228 | + expect(suite.passed).toBe(1); |
| 229 | + expect(exitCode).toBe(0); |
| 230 | +}); |
| 231 | + |
| 232 | +test('Maestro step sessions track the running attempt across a retry', async () => { |
| 233 | + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-reporter-maestro-retry-')); |
| 234 | + fs.writeFileSync(path.join(root, '01-retry.yaml'), MAESTRO_FLOW); |
| 235 | + |
| 236 | + // The first attempt's `back` fails, so attempt 1 fails at step 2 and attempt 2 passes. |
| 237 | + let backCalls = 0; |
| 238 | + const { hooks, suite } = await runMaestroSuiteThroughReporter({ |
| 239 | + root, |
| 240 | + requestId: 'suite-maestro-retry', |
| 241 | + flags: { retries: 1 }, |
| 242 | + invoke: async (req) => { |
| 243 | + if (req.command !== 'snapshot') { |
| 244 | + backCalls += 1; |
| 245 | + if (backCalls === 1) { |
| 246 | + return { ok: false, error: { code: 'COMMAND_FAILED', message: 'first attempt failed' } }; |
| 247 | + } |
| 248 | + } |
| 249 | + return maestroInvoke(req); |
| 250 | + }, |
| 251 | + }); |
| 252 | + |
| 253 | + const steps = hooks.filter((entry) => entry.hook === 'onTestStep').map((entry) => entry.value); |
| 254 | + |
| 255 | + // Every step event of attempt 1 carries attempt-1's session, and every step event of |
| 256 | + // attempt 2 carries attempt-2's. This is the identity the removed AsyncLocalStorage used to |
| 257 | + // supply, now threaded from the scheduler's attempt context through the Maestro observer. |
| 258 | + const attempts = steps.map((step) => step.attempt); |
| 259 | + const sessions = steps.map((step) => step.session); |
| 260 | + expect(new Set(attempts)).toEqual(new Set([1, 2])); |
| 261 | + for (const [index, step] of steps.entries()) { |
| 262 | + expect(step.session).toBe( |
| 263 | + `default:test:suite-maestro-retry:1-01-retry:attempt-${String(attempts[index])}`, |
| 264 | + ); |
| 265 | + } |
| 266 | + expect(new Set(sessions).size).toBe(2); |
| 267 | + expect(steps.every((step) => step.maxAttempts === 2)).toBe(true); |
| 268 | + |
| 269 | + // The retry result and the final result, both Maestro-produced. |
| 270 | + const results = hooks |
| 271 | + .filter((entry) => entry.hook === 'onTestResult') |
| 272 | + .map((entry) => entry.value); |
| 273 | + expect(results[0]).toMatchObject({ |
| 274 | + status: 'fail', |
| 275 | + attempt: 1, |
| 276 | + retrying: true, |
| 277 | + session: 'default:test:suite-maestro-retry:1-01-retry:attempt-1', |
| 278 | + }); |
| 279 | + expect(results[1]).toMatchObject({ |
| 280 | + status: 'pass', |
| 281 | + attempt: 2, |
| 282 | + retrying: undefined, |
| 283 | + session: 'default:test:suite-maestro-retry:1-01-retry:attempt-2', |
| 284 | + }); |
| 285 | + expect(suite.passed).toBe(1); |
| 286 | +}); |
0 commit comments