|
1 | | -import fs from 'node:fs'; |
2 | | -import path from 'node:path'; |
3 | | -import { onTestFinished, test } from 'vitest'; |
| 1 | +import { expect, test } from 'vitest'; |
4 | 2 | import assert from 'node:assert/strict'; |
| 3 | +import { AppError } from '@agent-device/kernel/errors'; |
5 | 4 | import { IOS_DEVICE, IOS_SIMULATOR, MACOS_DEVICE } from './device-fixtures.ts'; |
6 | 5 | import { |
| 6 | + diffComparableRunnerCacheMetadata, |
7 | 7 | resolveRunnerBundleBuildSettings, |
8 | 8 | resolveRunnerMaxConcurrentDestinationsFlag, |
9 | 9 | resolveRunnerSigningBuildSettings, |
10 | 10 | resolveRunnerPerformanceBuildSettings, |
11 | 11 | resolveRunnerSandboxBuildArgs, |
12 | 12 | resolveExpectedRunnerCacheMetadata, |
13 | 13 | } from '../runner-cache-metadata.ts'; |
14 | | -import { mkdtempForTestSync } from './tmp-dir.ts'; |
| 14 | +import { appleToolchainProbeResult, stubAppleToolchainProbes } from './apple-toolchain-fixtures.ts'; |
| 15 | + |
| 16 | +const runCmdSync = stubAppleToolchainProbes(); |
15 | 17 |
|
16 | 18 | test('resolveRunnerMaxConcurrentDestinationsFlag uses simulator flag for simulators', () => { |
17 | 19 | assert.equal( |
@@ -40,7 +42,10 @@ test('resolveRunnerSigningBuildSettings returns empty args without env overrides |
40 | 42 |
|
41 | 43 | test('resolveRunnerSigningBuildSettings disables signing for macOS desktop builds', () => { |
42 | 44 | assert.deepEqual( |
43 | | - resolveRunnerSigningBuildSettings({}, true, { platform: 'apple', appleOs: 'macos' }), |
| 45 | + resolveRunnerSigningBuildSettings({}, true, { |
| 46 | + platform: 'apple', |
| 47 | + appleOs: 'macos', |
| 48 | + }), |
44 | 49 | [ |
45 | 50 | 'CODE_SIGNING_ALLOWED=NO', |
46 | 51 | 'CODE_SIGNING_REQUIRED=NO', |
@@ -153,83 +158,229 @@ test('resolveRunnerBundleBuildSettings uses AGENT_DEVICE_IOS_BUNDLE_ID when prov |
153 | 158 | ); |
154 | 159 | }); |
155 | 160 |
|
156 | | -test('runner cache metadata fingerprints shared snapshot presentation sources', () => { |
157 | | - const root = mkdtempForTestSync('agent-device-runner-cache-fingerprint-'); |
158 | | - onTestFinished(() => fs.rmSync(root, { recursive: true, force: true })); |
159 | | - fs.writeFileSync(path.join(root, 'package.json'), JSON.stringify({ version: '0.0.0' })); |
160 | | - fs.mkdirSync(path.join(root, 'apple', 'runner', 'AgentDeviceRunner'), { recursive: true }); |
161 | | - fs.mkdirSync(path.join(root, 'apple', 'snapshot-presentation', 'Sources'), { recursive: true }); |
162 | | - fs.writeFileSync( |
163 | | - path.join(root, 'apple', 'runner', 'AgentDeviceRunner', 'Runner.swift'), |
164 | | - 'runner\n', |
165 | | - ); |
166 | | - const sharedSource = path.join( |
167 | | - root, |
168 | | - 'apple', |
169 | | - 'snapshot-presentation', |
170 | | - 'Sources', |
171 | | - 'Presentation.swift', |
172 | | - ); |
173 | | - fs.writeFileSync(sharedSource, 'shared-one\n'); |
174 | | - |
175 | | - const before = resolveExpectedRunnerCacheMetadata(IOS_SIMULATOR, root).runnerSourceFingerprint; |
176 | | - fs.writeFileSync(sharedSource, 'shared-two\n'); |
177 | | - const after = resolveExpectedRunnerCacheMetadata(IOS_SIMULATOR, root).runnerSourceFingerprint; |
178 | | - |
179 | | - assert.notEqual(after, before); |
180 | | -}); |
181 | | - |
182 | | -test('runner cache metadata ignores development-only SwiftPM trees but keeps runner unit tests', () => { |
183 | | - const root = mkdtempForTestSync('agent-device-runner-cache-source-roots-'); |
184 | | - onTestFinished(() => fs.rmSync(root, { recursive: true, force: true })); |
185 | | - fs.writeFileSync(path.join(root, 'package.json'), JSON.stringify({ version: '0.0.0' })); |
186 | | - |
187 | | - const runnerRoot = path.join(root, 'apple', 'runner', 'AgentDeviceRunner'); |
188 | | - const runnerUnitTest = path.join( |
189 | | - runnerRoot, |
190 | | - 'AgentDeviceRunnerUITests', |
191 | | - 'UnitTests', |
192 | | - 'Invariant.swift', |
193 | | - ); |
194 | | - const sharedRoot = path.join(root, 'apple', 'snapshot-presentation'); |
195 | | - fs.mkdirSync(path.dirname(runnerUnitTest), { recursive: true }); |
196 | | - fs.mkdirSync(path.join(sharedRoot, 'Sources'), { recursive: true }); |
197 | | - fs.writeFileSync(path.join(runnerRoot, 'Runner.swift'), 'runner\n'); |
198 | | - fs.writeFileSync(runnerUnitTest, 'unit-one\n'); |
199 | | - fs.writeFileSync(path.join(sharedRoot, 'Sources', 'Presentation.swift'), 'shared\n'); |
200 | | - |
201 | | - for (const directory of [ |
202 | | - 'Tests', |
203 | | - 'SnapshotPresentationConformance', |
204 | | - '.build', |
205 | | - '.swiftpm', |
206 | | - 'xcuserdata', |
207 | | - ]) { |
208 | | - const file = path.join(sharedRoot, directory, 'Ignored.swift'); |
209 | | - fs.mkdirSync(path.dirname(file), { recursive: true }); |
210 | | - fs.writeFileSync(file, 'ignored-one\n'); |
| 161 | +test('metadata diff names only the comparable keys that differ, with expected and actual', () => { |
| 162 | + const expected = resolveExpectedRunnerCacheMetadata(IOS_SIMULATOR); |
| 163 | + const actual = { |
| 164 | + ...expected, |
| 165 | + packageVersion: `${expected.packageVersion}-next`, |
| 166 | + xcodeBuildVersion: '17A100', |
| 167 | + runnerPerformanceBuildSettings: ['ENABLE_CODE_COVERAGE=YES'], |
| 168 | + artifacts: { |
| 169 | + xctestrunPath: '/tmp/derived/Runner.xctestrun', |
| 170 | + xctestrunMtimeMs: 1, |
| 171 | + xctestrunSize: 2, |
| 172 | + productPaths: [{ path: '/tmp/derived/Runner.app', mtimeMs: 1, size: 2 }], |
| 173 | + }, |
| 174 | + }; |
| 175 | + |
| 176 | + assert.deepEqual(diffComparableRunnerCacheMetadata(expected, actual), [ |
| 177 | + { |
| 178 | + key: 'runnerPerformanceBuildSettings', |
| 179 | + expected: JSON.stringify(expected.runnerPerformanceBuildSettings), |
| 180 | + actual: '["ENABLE_CODE_COVERAGE=YES"]', |
| 181 | + }, |
| 182 | + { key: 'xcodeBuildVersion', expected: '"17C52"', actual: '"17A100"' }, |
| 183 | + ]); |
| 184 | +}); |
| 185 | + |
| 186 | +test('metadata diff reports a key only one side carries as absent', () => { |
| 187 | + const expected = resolveExpectedRunnerCacheMetadata(IOS_SIMULATOR); |
| 188 | + const { sdkBuildVersion: _sdkBuildVersion, ...withoutSdkBuildVersion } = expected; |
| 189 | + |
| 190 | + assert.deepEqual( |
| 191 | + diffComparableRunnerCacheMetadata(expected, withoutSdkBuildVersion as typeof expected), |
| 192 | + [{ key: 'sdkBuildVersion', expected: '"23C53"', actual: '(absent)' }], |
| 193 | + ); |
| 194 | +}); |
| 195 | + |
| 196 | +test('metadata diff is empty for identical metadata', () => { |
| 197 | + const expected = resolveExpectedRunnerCacheMetadata(IOS_SIMULATOR); |
| 198 | + |
| 199 | + assert.deepEqual(diffComparableRunnerCacheMetadata(expected, { ...expected }), []); |
| 200 | +}); |
| 201 | + |
| 202 | +test('metadata diff elides an over-long value in the middle so both ends stay comparable', () => { |
| 203 | + const expected = resolveExpectedRunnerCacheMetadata(IOS_SIMULATOR); |
| 204 | + const longSetting = (suffix: string) => [`${'A'.repeat(400)}=${suffix}`]; |
| 205 | + |
| 206 | + const [difference] = diffComparableRunnerCacheMetadata( |
| 207 | + { ...expected, runnerBundleBuildSettings: longSetting('one') }, |
| 208 | + { ...expected, runnerBundleBuildSettings: longSetting('two') }, |
| 209 | + ); |
| 210 | + |
| 211 | + assert.equal(difference?.key, 'runnerBundleBuildSettings'); |
| 212 | + assert.ok((difference?.expected.length ?? 0) <= 300); |
| 213 | + assert.ok(difference?.expected.startsWith('["AAA')); |
| 214 | + assert.ok(difference?.expected.endsWith('=one"]')); |
| 215 | + assert.ok(difference?.actual.endsWith('=two"]')); |
| 216 | +}); |
| 217 | + |
| 218 | +function unavailableProbes(): { probe: string; reason: string }[] { |
| 219 | + try { |
| 220 | + resolveExpectedRunnerCacheMetadata(IOS_SIMULATOR); |
| 221 | + return []; |
| 222 | + } catch (error) { |
| 223 | + assert.ok(error instanceof AppError); |
| 224 | + assert.equal(error.details?.reason, 'apple_toolchain_probe_unavailable'); |
| 225 | + const probes = error.details?.probes as { probe: string; reason: string }[]; |
| 226 | + return probes.map(({ probe, reason }) => ({ probe, reason })); |
211 | 227 | } |
| 228 | +} |
| 229 | + |
| 230 | +test('a timed-out probe leaves the toolchain unavailable instead of a comparable value', () => { |
| 231 | + runCmdSync.mockImplementation((command: string, args: readonly string[]) => { |
| 232 | + if (command === 'xcodebuild') { |
| 233 | + throw new AppError('COMMAND_FAILED', 'xcodebuild timed out after 5000ms', { |
| 234 | + timeoutMs: 5_000, |
| 235 | + }); |
| 236 | + } |
| 237 | + return appleToolchainProbeResult(command, args); |
| 238 | + }); |
| 239 | + |
| 240 | + assert.deepEqual(unavailableProbes(), [{ probe: 'xcodebuild -version', reason: 'probe_error' }]); |
| 241 | +}); |
| 242 | + |
| 243 | +test('a failing probe reports its exit status rather than a fabricated SDK version', () => { |
| 244 | + runCmdSync.mockImplementation((command: string, args: readonly string[]) => |
| 245 | + command === 'xcrun' |
| 246 | + ? { |
| 247 | + exitCode: 70, |
| 248 | + stdout: '', |
| 249 | + stderr: 'xcrun: error: SDK cannot be located\n', |
| 250 | + } |
| 251 | + : appleToolchainProbeResult(command, args), |
| 252 | + ); |
212 | 253 |
|
213 | | - const before = resolveExpectedRunnerCacheMetadata(IOS_SIMULATOR, root).runnerSourceFingerprint; |
214 | | - for (const directory of [ |
215 | | - 'Tests', |
216 | | - 'SnapshotPresentationConformance', |
217 | | - '.build', |
218 | | - '.swiftpm', |
219 | | - 'xcuserdata', |
220 | | - ]) { |
221 | | - fs.writeFileSync(path.join(sharedRoot, directory, 'Ignored.swift'), 'ignored-two\n'); |
| 254 | + assert.deepEqual(unavailableProbes(), [ |
| 255 | + { |
| 256 | + probe: 'xcrun --sdk iphonesimulator --show-sdk-version', |
| 257 | + reason: 'nonzero_exit', |
| 258 | + }, |
| 259 | + { |
| 260 | + probe: 'xcrun --sdk iphonesimulator --show-sdk-build-version', |
| 261 | + reason: 'nonzero_exit', |
| 262 | + }, |
| 263 | + ]); |
| 264 | +}); |
| 265 | + |
| 266 | +test('unrecognized xcodebuild output is unavailable, not a partially parsed fingerprint', () => { |
| 267 | + runCmdSync.mockImplementation((command: string, args: readonly string[]) => |
| 268 | + command === 'xcodebuild' |
| 269 | + ? { |
| 270 | + exitCode: 0, |
| 271 | + stdout: 'xcode-select: error: tool not configured\n', |
| 272 | + stderr: '', |
| 273 | + } |
| 274 | + : appleToolchainProbeResult(command, args), |
| 275 | + ); |
| 276 | + |
| 277 | + assert.deepEqual(unavailableProbes(), [ |
| 278 | + { probe: 'xcodebuild -version', reason: 'unparsable_output' }, |
| 279 | + ]); |
| 280 | +}); |
| 281 | + |
| 282 | +test('an empty probe answer is unavailable rather than an empty cache key field', () => { |
| 283 | + runCmdSync.mockImplementation((command: string, args: readonly string[]) => |
| 284 | + command === 'xcrun' && args.includes('--show-sdk-build-version') |
| 285 | + ? { exitCode: 0, stdout: '\n', stderr: '' } |
| 286 | + : appleToolchainProbeResult(command, args), |
| 287 | + ); |
| 288 | + |
| 289 | + assert.deepEqual(unavailableProbes(), [ |
| 290 | + { |
| 291 | + probe: 'xcrun --sdk iphonesimulator --show-sdk-build-version', |
| 292 | + reason: 'empty_output', |
| 293 | + }, |
| 294 | + ]); |
| 295 | +}); |
| 296 | + |
| 297 | +test('an unavailable toolchain fails the cache decision with a retriable typed error', () => { |
| 298 | + runCmdSync.mockImplementation(() => { |
| 299 | + throw new AppError('COMMAND_FAILED', 'xcodebuild timed out after 5000ms', {}); |
| 300 | + }); |
| 301 | + |
| 302 | + try { |
| 303 | + resolveExpectedRunnerCacheMetadata(IOS_SIMULATOR); |
| 304 | + assert.fail('expected an unavailable toolchain to fail the cache decision'); |
| 305 | + } catch (error) { |
| 306 | + assert.ok(error instanceof AppError); |
| 307 | + assert.equal(error.code, 'COMMAND_FAILED'); |
| 308 | + assert.equal(error.details?.retriable, true); |
| 309 | + expect(error.message).toContain('xcodebuild -version'); |
| 310 | + expect(String(error.details?.hint)).toContain('xcode-select'); |
222 | 311 | } |
223 | | - const afterIgnoredChanges = resolveExpectedRunnerCacheMetadata( |
224 | | - IOS_SIMULATOR, |
225 | | - root, |
226 | | - ).runnerSourceFingerprint; |
227 | | - assert.equal(afterIgnoredChanges, before); |
228 | | - |
229 | | - fs.writeFileSync(runnerUnitTest, 'unit-two\n'); |
230 | | - const afterRunnerTestChange = resolveExpectedRunnerCacheMetadata( |
231 | | - IOS_SIMULATOR, |
232 | | - root, |
233 | | - ).runnerSourceFingerprint; |
234 | | - assert.notEqual(afterRunnerTestChange, afterIgnoredChanges); |
| 312 | +}); |
| 313 | + |
| 314 | +test('an unavailable probe never reaches cache metadata, and is not memoized as one', () => { |
| 315 | + runCmdSync.mockImplementation(() => { |
| 316 | + throw new AppError('COMMAND_FAILED', 'xcodebuild timed out after 5000ms', {}); |
| 317 | + }); |
| 318 | + expect(() => resolveExpectedRunnerCacheMetadata(IOS_SIMULATOR)).toThrow( |
| 319 | + /Could not read the Xcode toolchain versions/, |
| 320 | + ); |
| 321 | + |
| 322 | + runCmdSync.mockImplementation(appleToolchainProbeResult); |
| 323 | + const metadata = resolveExpectedRunnerCacheMetadata(IOS_SIMULATOR); |
| 324 | + |
| 325 | + assert.equal(metadata.xcodeVersion, '26.2'); |
| 326 | + assert.equal(metadata.xcodeBuildVersion, '17C52'); |
| 327 | + assert.equal(metadata.sdkVersion, '26.2'); |
| 328 | + assert.equal(metadata.sdkBuildVersion, '23C53'); |
| 329 | +}); |
| 330 | + |
| 331 | +test('a malformed xcodebuild answer is not memoized: the next request re-probes and recovers', () => { |
| 332 | + runCmdSync.mockImplementation((command: string, args: readonly string[]) => |
| 333 | + command === 'xcodebuild' |
| 334 | + ? { |
| 335 | + exitCode: 0, |
| 336 | + stdout: 'xcode-select: error: tool not configured\n', |
| 337 | + stderr: '', |
| 338 | + } |
| 339 | + : appleToolchainProbeResult(command, args), |
| 340 | + ); |
| 341 | + assert.deepEqual(unavailableProbes(), [ |
| 342 | + { probe: 'xcodebuild -version', reason: 'unparsable_output' }, |
| 343 | + ]); |
| 344 | + const probeCallsWhileMalformed = runCmdSync.mock.calls.length; |
| 345 | + |
| 346 | + runCmdSync.mockImplementation(appleToolchainProbeResult); |
| 347 | + const metadata = resolveExpectedRunnerCacheMetadata(IOS_SIMULATOR); |
| 348 | + |
| 349 | + assert.equal(metadata.xcodeVersion, '26.2'); |
| 350 | + assert.equal(metadata.xcodeBuildVersion, '17C52'); |
| 351 | + expect(runCmdSync.mock.calls.slice(probeCallsWhileMalformed).map(([command]) => command)).toEqual( |
| 352 | + ['xcodebuild', 'xcrun', 'xcrun'], |
| 353 | + ); |
| 354 | +}); |
| 355 | + |
| 356 | +test('only a complete, parsed toolchain fingerprint is memoized', () => { |
| 357 | + runCmdSync.mockImplementation((command: string, args: readonly string[]) => |
| 358 | + command === 'xcrun' && args.includes('--show-sdk-build-version') |
| 359 | + ? { exitCode: 0, stdout: '\n', stderr: '' } |
| 360 | + : appleToolchainProbeResult(command, args), |
| 361 | + ); |
| 362 | + assert.deepEqual(unavailableProbes(), [ |
| 363 | + { |
| 364 | + probe: 'xcrun --sdk iphonesimulator --show-sdk-build-version', |
| 365 | + reason: 'empty_output', |
| 366 | + }, |
| 367 | + ]); |
| 368 | + |
| 369 | + runCmdSync.mockImplementation(appleToolchainProbeResult); |
| 370 | + runCmdSync.mockClear(); |
| 371 | + const first = resolveExpectedRunnerCacheMetadata(IOS_SIMULATOR); |
| 372 | + // The healthy xcodebuild answer from the failed round was not kept either: all three re-run. |
| 373 | + expect(runCmdSync.mock.calls.map(([command]) => command)).toEqual([ |
| 374 | + 'xcodebuild', |
| 375 | + 'xcrun', |
| 376 | + 'xcrun', |
| 377 | + ]); |
| 378 | + |
| 379 | + runCmdSync.mockClear(); |
| 380 | + const second = resolveExpectedRunnerCacheMetadata(IOS_SIMULATOR); |
| 381 | + expect(runCmdSync).not.toHaveBeenCalled(); |
| 382 | + assert.deepEqual( |
| 383 | + [second.xcodeVersion, second.xcodeBuildVersion, second.sdkVersion, second.sdkBuildVersion], |
| 384 | + [first.xcodeVersion, first.xcodeBuildVersion, first.sdkVersion, first.sdkBuildVersion], |
| 385 | + ); |
235 | 386 | }); |
0 commit comments