Skip to content

Commit a4c2075

Browse files
author
Jonah Williams
authored
[devicelab] reduce iterations, uninstall at end, and use --application-binary in all startup tests (#68516)
These devicelab tests are incredibly flaky. Currently they will build & install the same application 15 times. This causes the temp storage to fill up on android, and has a good chance of flaking on iOS due to install issues. The change from 3 to 15 increased total test time for 2 -> 12 minutes on iOS, or roughly 5 more test equivalents for every single iOS test. Reduce the iteration count back to 5 Uninstall the app after each run so temp storage does not fill up.
1 parent 98c519d commit a4c2075

1 file changed

Lines changed: 38 additions & 6 deletions

File tree

‎dev/devicelab/lib/tasks/perf_tests.dart‎

Lines changed: 38 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -454,24 +454,56 @@ class StartupTest {
454454

455455
Future<TaskResult> run() async {
456456
return await inDirectory<TaskResult>(testDirectory, () async {
457-
final String deviceId = (await devices.workingDevice).deviceId;
458-
await flutter('packages', options: <String>['get']);
459-
460-
const int iterations = 15;
457+
final Device device = await devices.workingDevice;
458+
const int iterations = 5;
461459
final List<Map<String, dynamic>> results = <Map<String, dynamic>>[];
462-
for (int i = 0; i < iterations; ++i) {
460+
461+
section('Building application');
462+
String applicationBinaryPath;
463+
switch (deviceOperatingSystem) {
464+
case DeviceOperatingSystem.android:
465+
await flutter('build', options: <String>[
466+
'apk',
467+
'-v',
468+
'--profile',
469+
'--target-platform=android-arm,android-arm64',
470+
]);
471+
applicationBinaryPath = '$testDirectory/build/app/outputs/flutter-apk/app-profile.apk';
472+
break;
473+
case DeviceOperatingSystem.ios:
474+
await flutter('build', options: <String>[
475+
'ios',
476+
'-v',
477+
'--profile',
478+
]);
479+
applicationBinaryPath = '$testDirectory/build/ios/iphoneos/Runner.app';
480+
break;
481+
case DeviceOperatingSystem.fuchsia:
482+
case DeviceOperatingSystem.fake:
483+
break;
484+
}
485+
486+
for (int i = 0; i < iterations; i += 1) {
463487
await flutter('run', options: <String>[
464488
'--no-android-gradle-daemon',
465489
'--verbose',
466490
'--profile',
467491
'--trace-startup',
468492
'-d',
469-
deviceId,
493+
device.deviceId,
494+
if (applicationBinaryPath != null)
495+
'--use-application-binary=$applicationBinaryPath',
470496
]);
471497
final Map<String, dynamic> data = json.decode(
472498
file('$testDirectory/build/start_up_info.json').readAsStringSync(),
473499
) as Map<String, dynamic>;
474500
results.add(data);
501+
502+
await flutter('install', options: <String>[
503+
'--uninstall-only',
504+
'-d',
505+
device.deviceId,
506+
]);
475507
}
476508

477509
final Map<String, dynamic> averageResults = _average(results, iterations);

0 commit comments

Comments
 (0)