|
8 | 8 | makeRunnerArtifact, |
9 | 9 | createTestRequestCancellation, |
10 | 10 | makeRunnerSession, |
| 11 | + runnerConnectFailure, |
11 | 12 | } from './runner-session-fixtures.ts'; |
12 | 13 |
|
13 | 14 | const { |
@@ -110,3 +111,177 @@ test('a prepare deadline spent during boot keeps the restored artifact and retri |
110 | 111 | vi.useRealTimers(); |
111 | 112 | } |
112 | 113 | }); |
| 114 | + |
| 115 | +// The bad-cache recovery decision: a restored artifact whose runner refuses the connection is |
| 116 | +// indicted, wiped for the run, and replaced by one forced rebuild — under one admission token |
| 117 | +// owned by the prepare loop, so the rebuild is the retry a fence governs rather than a fresh |
| 118 | +// start that slips past it (#3220). |
| 119 | + |
| 120 | +test('prepareIosRunner marks a bad restored artifact and rebuilds once after health failure', async () => { |
| 121 | + const fixtures = makeBadCacheRecoveryFixtures(); |
| 122 | + |
| 123 | + mockEnsureRunnerSession |
| 124 | + .mockResolvedValueOnce(fixtures.restoredSession) |
| 125 | + .mockResolvedValueOnce(fixtures.rebuiltSession); |
| 126 | + mockExecuteRunnerCommandWithSession |
| 127 | + .mockRejectedValueOnce(runnerConnectFailure('runner_connect_refused')) |
| 128 | + .mockResolvedValueOnce({ uptimeMs: 42 }); |
| 129 | + |
| 130 | + const result = await prepareIosRunner(IOS_SIMULATOR, { |
| 131 | + healthTimeoutMs: 90_000, |
| 132 | + buildTimeoutMs: 300_000, |
| 133 | + }); |
| 134 | + |
| 135 | + assertRecoveredPrepareResult(result); |
| 136 | + assertBadCacheRecoverySideEffects(fixtures); |
| 137 | + assertRecoveredPrepareDiagnostics(); |
| 138 | +}); |
| 139 | + |
| 140 | +test('prepareIosRunner invalidates rebuilt sessions when bad-cache recovery health fails', async () => { |
| 141 | + const restoredArtifact = makeRunnerArtifact({ |
| 142 | + xctestrunPath: '/tmp/restored.xctestrun', |
| 143 | + cache: 'exact', |
| 144 | + artifact: 'valid', |
| 145 | + }); |
| 146 | + const rebuiltArtifact = makeRunnerArtifact({ |
| 147 | + xctestrunPath: '/tmp/rebuilt.xctestrun', |
| 148 | + cache: 'miss', |
| 149 | + artifact: 'rebuilt', |
| 150 | + }); |
| 151 | + const restoredSession = makeRunnerSession({ |
| 152 | + port: 8100, |
| 153 | + xctestrunPath: restoredArtifact.xctestrunPath, |
| 154 | + xctestrunArtifact: restoredArtifact, |
| 155 | + }); |
| 156 | + const rebuiltSession = makeRunnerSession({ |
| 157 | + port: 8101, |
| 158 | + xctestrunPath: rebuiltArtifact.xctestrunPath, |
| 159 | + xctestrunArtifact: rebuiltArtifact, |
| 160 | + }); |
| 161 | + |
| 162 | + mockEnsureRunnerSession |
| 163 | + .mockResolvedValueOnce(restoredSession) |
| 164 | + .mockResolvedValueOnce(rebuiltSession); |
| 165 | + mockExecuteRunnerCommandWithSession |
| 166 | + .mockRejectedValueOnce(runnerConnectFailure('runner_endpoint_probe_exhausted')) |
| 167 | + .mockRejectedValueOnce(new AppError('COMMAND_FAILED', 'Runner health timed out')); |
| 168 | + |
| 169 | + await assert.rejects( |
| 170 | + () => prepareIosRunner(IOS_SIMULATOR, { healthTimeoutMs: 90_000 }), |
| 171 | + (error: unknown) => { |
| 172 | + assert.ok(error instanceof AppError); |
| 173 | + assert.equal(error.message, 'artifact restored but runner did not connect'); |
| 174 | + assert.equal(error.details?.restoredFailureReason, 'Runner endpoint probe failed'); |
| 175 | + assert.equal(error.details?.xctestrunPath, '/tmp/rebuilt.xctestrun'); |
| 176 | + assert.equal(error.details?.artifact, 'rebuilt'); |
| 177 | + assert.equal(error.details?.cache, 'miss'); |
| 178 | + return true; |
| 179 | + }, |
| 180 | + ); |
| 181 | + |
| 182 | + assert.deepEqual(mockInvalidateRunnerSession.mock.calls, [ |
| 183 | + [restoredSession, 'prepare_cached_runner_health_failed'], |
| 184 | + [rebuiltSession, 'prepare_rebuilt_runner_health_failed'], |
| 185 | + ]); |
| 186 | + assert.deepEqual(mockMarkRunnerXctestrunArtifactBadForRun.mock.calls[0], [ |
| 187 | + restoredArtifact, |
| 188 | + 'Runner endpoint probe failed', |
| 189 | + ]); |
| 190 | +}); |
| 191 | + |
| 192 | +function makeBadCacheRecoveryFixtures() { |
| 193 | + const restoredArtifact = makeRunnerArtifact({ |
| 194 | + xctestrunPath: '/tmp/restored.xctestrun', |
| 195 | + cache: 'exact', |
| 196 | + artifact: 'valid', |
| 197 | + }); |
| 198 | + const rebuiltArtifact = makeRunnerArtifact({ |
| 199 | + xctestrunPath: '/tmp/rebuilt.xctestrun', |
| 200 | + cache: 'miss', |
| 201 | + artifact: 'rebuilt', |
| 202 | + buildMs: 123, |
| 203 | + }); |
| 204 | + const restoredSession = makeRunnerSession({ |
| 205 | + port: 8100, |
| 206 | + xctestrunPath: restoredArtifact.xctestrunPath, |
| 207 | + xctestrunArtifact: restoredArtifact, |
| 208 | + }); |
| 209 | + const rebuiltSession = makeRunnerSession({ |
| 210 | + port: 8101, |
| 211 | + xctestrunPath: rebuiltArtifact.xctestrunPath, |
| 212 | + xctestrunArtifact: rebuiltArtifact, |
| 213 | + }); |
| 214 | + |
| 215 | + return { restoredArtifact, restoredSession, rebuiltSession }; |
| 216 | +} |
| 217 | + |
| 218 | +function assertRecoveredPrepareResult(result: Awaited<ReturnType<typeof prepareIosRunner>>): void { |
| 219 | + assert.deepEqual(result, { |
| 220 | + runner: { uptimeMs: 42 }, |
| 221 | + cache: 'miss', |
| 222 | + artifact: 'rebuilt', |
| 223 | + buildMs: 123, |
| 224 | + connectMs: result.connectMs, |
| 225 | + healthCheckMs: result.healthCheckMs, |
| 226 | + xctestrunPath: '/tmp/rebuilt.xctestrun', |
| 227 | + recoveryReason: 'Runner did not accept connection', |
| 228 | + }); |
| 229 | + assert.equal(result.failureReason, undefined); |
| 230 | + assert.equal(result.connectMs >= 0, true); |
| 231 | + assert.equal(result.healthCheckMs >= 0, true); |
| 232 | +} |
| 233 | + |
| 234 | +function assertBadCacheRecoverySideEffects( |
| 235 | + fixtures: ReturnType<typeof makeBadCacheRecoveryFixtures>, |
| 236 | +): void { |
| 237 | + assert.deepEqual(mockInvalidateRunnerSession.mock.calls[0], [ |
| 238 | + fixtures.restoredSession, |
| 239 | + 'prepare_cached_runner_health_failed', |
| 240 | + ]); |
| 241 | + assert.deepEqual(mockMarkRunnerXctestrunArtifactBadForRun.mock.calls[0], [ |
| 242 | + fixtures.restoredArtifact, |
| 243 | + 'Runner did not accept connection', |
| 244 | + ]); |
| 245 | + const firstCallOptions = mockEnsureRunnerSession.mock.calls[0]?.[1] as |
| 246 | + | { startAdmission?: unknown } |
| 247 | + | undefined; |
| 248 | + const retryOptions = mockEnsureRunnerSession.mock.calls[1]?.[1] as |
| 249 | + | Record<string, unknown> |
| 250 | + | undefined; |
| 251 | + assert.deepEqual(retryOptions, { |
| 252 | + healthTimeoutMs: 90_000, |
| 253 | + buildTimeoutMs: 300_000, |
| 254 | + cleanStaleBundles: true, |
| 255 | + forceRunnerXctestrunRebuild: true, |
| 256 | + startAdmission: firstCallOptions?.startAdmission, |
| 257 | + }); |
| 258 | + // Both attempts of the prepare loop share one admission token: the retry |
| 259 | + // is the replacement build for the first attempt, not a newcomer. |
| 260 | + assert.ok(firstCallOptions?.startAdmission); |
| 261 | + assert.equal(mockExecuteRunnerCommandWithSession.mock.calls.length, 2); |
| 262 | + assert.equal(mockExecuteRunnerCommandWithSession.mock.calls[0]?.[2].command, 'uptime'); |
| 263 | + assert.equal(mockExecuteRunnerCommandWithSession.mock.calls[0]?.[4], 90_000); |
| 264 | + assert.equal(mockExecuteRunnerCommandWithSession.mock.calls[1]?.[1], fixtures.rebuiltSession); |
| 265 | +} |
| 266 | + |
| 267 | +function assertRecoveredPrepareDiagnostics(): void { |
| 268 | + assert.ok( |
| 269 | + mockEmitDiagnostic.mock.calls.some( |
| 270 | + ([event]) => event.phase === 'ios_runner_prepare_bad_cache_recovered', |
| 271 | + ), |
| 272 | + ); |
| 273 | + const prepareDiagnostic = mockEmitDiagnostic.mock.calls.find( |
| 274 | + ([event]) => event.phase === 'apple_runner_prepare', |
| 275 | + )?.[0]; |
| 276 | + assert.ok(prepareDiagnostic); |
| 277 | + assert.equal(prepareDiagnostic.level, 'info'); |
| 278 | + assert.equal(prepareDiagnostic.data?.cache, 'miss'); |
| 279 | + assert.equal(prepareDiagnostic.data?.artifact, 'rebuilt'); |
| 280 | + assert.equal(prepareDiagnostic.data?.xctestrunPath, '/tmp/rebuilt.xctestrun'); |
| 281 | + assert.equal(prepareDiagnostic.data?.recoveryReason, 'Runner did not accept connection'); |
| 282 | + assert.equal(prepareDiagnostic.data?.failureReason, undefined); |
| 283 | + assert.deepEqual(prepareDiagnostic.data?.timingContainment, { |
| 284 | + connectMs: ['buildMs'], |
| 285 | + healthCheckMs: [], |
| 286 | + }); |
| 287 | +} |
0 commit comments