Add a E2E based performance test case #61509
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| // Copyright 2014 The Flutter Authors. All rights reserved. | ||
| // Use of this source code is governed by a BSD-style license that can be | ||
| // found in the LICENSE file. | ||
|
|
||
| // The test should be run as: | ||
| // flutter drive -t test/cull_opacity_perf_e2e.dart --driver test_driver/e2e_test.dart --trace-startup --profile | ||
|
|
||
| import 'package:macrobenchmarks/common.dart'; | ||
|
|
||
| import 'util.dart'; | ||
|
|
||
| Future<void> main() async { | ||
| macroPerfTestE2E( | ||
| 'cull_opacity_perf', | ||
| kCullOpacityRouteName, | ||
| pageDelay: const Duration(seconds: 1), | ||
| duration: const Duration(seconds: 10), | ||
| timeout: const Duration(seconds: 45), | ||
| ); | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,220 @@ | ||
| // Copyright 2014 The Flutter Authors. All rights reserved. | ||
|
||
| // Use of this source code is governed by a BSD-style license that can be | ||
| // found in the LICENSE file. | ||
|
|
||
| import 'dart:async'; | ||
| import 'dart:ui'; | ||
|
|
||
| import 'package:flutter/scheduler.dart'; | ||
| import 'package:flutter_test/flutter_test.dart'; | ||
| import 'package:flutter/widgets.dart'; | ||
|
|
||
| import 'package:macrobenchmarks/common.dart'; | ||
| import 'package:e2e/e2e.dart'; | ||
| import 'package:macrobenchmarks/main.dart' as app; | ||
|
|
||
| /// The maximum amount of time considered safe to spend for a frame's build | ||
| /// phase. Anything past that is in the danger of missing the frame as 60FPS. | ||
| /// | ||
| /// Changing this doesn't re-evaluate existing summary. | ||
| Duration kBuildBudget = const Duration(milliseconds: 16); | ||
CareF marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
| // TODO(CareF): Automatically calculate the refresh budget (#61958) | ||
|
|
||
| typedef ControlCallback = Future<void> Function(WidgetController controller); | ||
|
|
||
| void macroPerfTestE2E( | ||
| String testName, | ||
| String routeName, { | ||
| Duration pageDelay, | ||
| Duration duration = const Duration(seconds: 3), | ||
| Duration timeout = const Duration(seconds: 30), | ||
| ControlCallback body, | ||
| ControlCallback setup, | ||
| }) { | ||
| assert(() { | ||
| debugPrint(kDebugWarning); | ||
| return true; | ||
| }()); | ||
| final WidgetsBinding _binding = E2EWidgetsFlutterBinding.ensureInitialized(); | ||
| assert(_binding is E2EWidgetsFlutterBinding); | ||
| final E2EWidgetsFlutterBinding binding = _binding as E2EWidgetsFlutterBinding; | ||
| binding.framePolicy = LiveTestWidgetsFlutterBindingFramePolicy.benchmarkLive; | ||
|
|
||
| testWidgets(testName, (WidgetTester tester) async { | ||
| assert((tester.binding as LiveTestWidgetsFlutterBinding).framePolicy == | ||
| LiveTestWidgetsFlutterBindingFramePolicy.fullyLive); | ||
| app.main(); | ||
| await tester.pumpAndSettle(); | ||
|
|
||
| // The slight initial delay avoids starting the timing during a | ||
| // period of increased load on the device. Without this delay, the | ||
| // benchmark has greater noise. | ||
| // See: https://github.com/flutter/flutter/issues/19434 | ||
| await tester.binding.delayed(const Duration(microseconds: 250)); | ||
|
|
||
| final Finder scrollable = | ||
| find.byKey(const ValueKey<String>(kScrollableName)); | ||
| expect(scrollable, findsOneWidget); | ||
| final Finder button = | ||
| find.byKey(ValueKey<String>(routeName), skipOffstage: false); | ||
| await tester.ensureVisible(button); | ||
| expect(button, findsOneWidget); | ||
| await tester.pumpAndSettle(); | ||
| await tester.tap(button); | ||
|
|
||
| if (pageDelay != null) { | ||
| // Wait for the page to load | ||
| await tester.binding.delayed(pageDelay); | ||
| } | ||
|
|
||
| if (setup != null) { | ||
| await setup(tester); | ||
| } | ||
|
|
||
| await watchPerformance(binding, () async { | ||
| final Future<void> durationFuture = tester.binding.delayed(duration); | ||
| if (body != null) { | ||
| await body(tester); | ||
| } | ||
| await durationFuture; | ||
| }); | ||
| }, semanticsEnabled: false, timeout: Timeout(timeout)); | ||
| } | ||
|
|
||
| Future<void> watchPerformance( | ||
| E2EWidgetsFlutterBinding binding, | ||
| Future<void> action(), | ||
| ) async { | ||
| final List<FrameTiming> frameTimings = <FrameTiming>[]; | ||
| final TimingsCallback watcher = frameTimings.addAll; | ||
| binding.addTimingsCallback(watcher); | ||
| await action(); | ||
| binding.removeTimingsCallback(watcher); | ||
| // TODO(CareF): determine if it's running on firebase and report metric online | ||
| final FrameTimingSummarizer frameTimes = FrameTimingSummarizer(frameTimings); | ||
| binding.reportData = <String, dynamic>{'performance': frameTimes.summary}; | ||
| } | ||
|
|
||
| /// This class and summarizes a list of [FrameTiming] for the performance | ||
| /// metrics. | ||
| class FrameTimingSummarizer { | ||
| factory FrameTimingSummarizer(List<FrameTiming> data) { | ||
| assert(data != null); | ||
| assert(data.isNotEmpty); | ||
| final List<Duration> frameBuildTime = List<Duration>.unmodifiable( | ||
| data.map<Duration>((FrameTiming datum) => datum.buildDuration), | ||
| ); | ||
| final List<Duration> frameBuildTimeSorted = List<Duration>.from(frameBuildTime)..sort(); | ||
| final List<Duration> frameRasterizerTime = List<Duration>.unmodifiable( | ||
| data.map<Duration>((FrameTiming datum) => datum.rasterDuration), | ||
| ); | ||
| final List<Duration> frameRasterizerTimeSorted = List<Duration>.from(frameBuildTime)..sort(); | ||
| final Duration Function(Duration, Duration) add = (Duration a, Duration b) => a + b; | ||
| return FrameTimingSummarizer._( | ||
| frameBuildTime: frameBuildTime, | ||
| frameRasterizerTime: frameRasterizerTime, | ||
| // This avarage calculation is microsecond precision, which is fine | ||
| // because typical values of these times are milliseconds. | ||
| averageFrameBuildTime: frameBuildTime.reduce(add) ~/ data.length, | ||
| p90FrameBuildTime: _findPercentile(frameBuildTimeSorted, 0.90), | ||
| p99FrameBuildTime: _findPercentile(frameBuildTimeSorted, 0.99), | ||
| worstFrameBuildTime: frameBuildTimeSorted.last, | ||
| missedFrameBuildBudget: _countExceed(frameBuildTimeSorted, kBuildBudget), | ||
| averageFrameRasterizerTime: frameRasterizerTime.reduce(add) ~/ data.length, | ||
| p90FrameRasterizerTime: _findPercentile(frameRasterizerTimeSorted, 0.90), | ||
| p99FrameRasterizerTime: _findPercentile(frameRasterizerTimeSorted, 0.90), | ||
| worstFrameRasterizerTime: frameRasterizerTimeSorted.last, | ||
| missedFrameRasterizerBudget: _countExceed(frameRasterizerTimeSorted, kBuildBudget), | ||
| ); | ||
| } | ||
|
|
||
|
Contributor
There was a problem hiding this comment. Nit: when a function has a lot of arguments, Flutter usually put them under
Contributor
Author
There was a problem hiding this comment.
|
||
| const FrameTimingSummarizer._({ | ||
| @required this.frameBuildTime, | ||
| @required this.frameRasterizerTime, | ||
| @required this.averageFrameBuildTime, | ||
| @required this.p90FrameBuildTime, | ||
| @required this.p99FrameBuildTime, | ||
| @required this.worstFrameBuildTime, | ||
| @required this.missedFrameBuildBudget, | ||
| @required this.averageFrameRasterizerTime, | ||
| @required this.p90FrameRasterizerTime, | ||
| @required this.p99FrameRasterizerTime, | ||
| @required this.worstFrameRasterizerTime, | ||
| @required this.missedFrameRasterizerBudget | ||
| }); | ||
|
|
||
| /// List of frame build time in microseconds | ||
| final List<Duration> frameBuildTime; | ||
|
|
||
| /// List of frame rasterizer time in microseconds | ||
| final List<Duration> frameRasterizerTime; | ||
|
|
||
| /// The average value of [frameBuildTime] in milliseconds. | ||
| final Duration averageFrameBuildTime; | ||
|
|
||
| /// The 90-th percentile value of [frameBuildTime] in milliseconds | ||
| final Duration p90FrameBuildTime; | ||
|
|
||
| /// The 99-th percentile value of [frameBuildTime] in milliseconds | ||
| final Duration p99FrameBuildTime; | ||
|
|
||
| /// The largest value of [frameBuildTime] in milliseconds | ||
| final Duration worstFrameBuildTime; | ||
|
|
||
| /// Number of items in [frameBuildTime] that's greater than [kBuildBudget] | ||
| final int missedFrameBuildBudget; | ||
|
|
||
| /// The average value of [frameRasterizerTime] in milliseconds. | ||
| final Duration averageFrameRasterizerTime; | ||
|
|
||
| /// The 90-th percentile value of [frameRasterizerTime] in milliseconds. | ||
| final Duration p90FrameRasterizerTime; | ||
|
|
||
| /// The 99-th percentile value of [frameRasterizerTime] in milliseconds. | ||
| final Duration p99FrameRasterizerTime; | ||
|
|
||
| /// The largest value of [frameRasterizerTime] in milliseconds. | ||
| final Duration worstFrameRasterizerTime; | ||
|
|
||
| /// Number of items in [frameRasterizerTime] that's greater than [kBuildBudget] | ||
| final int missedFrameRasterizerBudget; | ||
|
|
||
| Map<String, dynamic> get summary => <String, dynamic>{ | ||
| 'average_frame_build_time_millis': | ||
| averageFrameBuildTime.inMicroseconds / 1E3, | ||
| '90th_percentile_frame_build_time_millis': | ||
| p90FrameBuildTime.inMicroseconds / 1E3, | ||
| '99th_percentile_frame_build_time_millis': | ||
| p99FrameBuildTime.inMicroseconds / 1E3, | ||
| 'worst_frame_build_time_millis': | ||
| worstFrameBuildTime.inMicroseconds / 1E3, | ||
| 'missed_frame_build_budget_count': missedFrameBuildBudget, | ||
| 'average_frame_rasterizer_time_millis': | ||
| averageFrameRasterizerTime.inMicroseconds / 1E3, | ||
| '90th_percentile_frame_rasterizer_time_millis': | ||
| p90FrameRasterizerTime.inMicroseconds / 1E3, | ||
| '99th_percentile_frame_rasterizer_time_millis': | ||
| p99FrameRasterizerTime.inMicroseconds / 1E3, | ||
| 'worst_frame_rasterizer_time_millis': | ||
| worstFrameRasterizerTime.inMicroseconds / 1E3, | ||
| 'missed_frame_rasterizer_budget_count': missedFrameRasterizerBudget, | ||
| 'frame_count': frameBuildTime.length, | ||
| 'frame_build_times': frameBuildTime | ||
| .map<int>((Duration datum) => datum.inMicroseconds).toList(), | ||
| 'frame_rasterizer_times': frameRasterizerTime | ||
| .map<int>((Duration datum) => datum.inMicroseconds).toList(), | ||
| }; | ||
| } | ||
|
|
||
| // The following helper functions require data sorted | ||
|
|
||
| // return the 100*p-th percentile of the data | ||
| T _findPercentile<T>(List<T> data, double p) { | ||
| assert(p >= 0 && p <= 1); | ||
| return data[((data.length - 1) * p).round()]; | ||
| } | ||
|
|
||
| // return the number of items in data that > threshold | ||
| int _countExceed<T extends Comparable<T>>(List<T> data, T threshold) { | ||
| return data.length - data.indexWhere((T datum) => datum.compareTo(threshold) > 0); | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| // Copyright 2014 The Flutter Authors. All rights reserved. | ||
| // Use of this source code is governed by a BSD-style license that can be | ||
| // found in the LICENSE file. | ||
|
|
||
| import 'dart:async'; | ||
| import 'dart:convert'; | ||
| import 'dart:io'; | ||
|
|
||
| import 'package:e2e/common.dart' as e2e; | ||
| import 'package:flutter_driver/flutter_driver.dart'; | ||
|
|
||
| import 'package:path/path.dart' as path; | ||
|
|
||
| const JsonEncoder _prettyEncoder = JsonEncoder.withIndent(' '); | ||
|
|
||
| /// Flutter Driver test output directory. | ||
| /// | ||
| /// Tests should write any output files to this directory. Defaults to the path | ||
| /// set in the FLUTTER_TEST_OUTPUTS_DIR environment variable, or `build` if | ||
| /// unset. | ||
| String testOutputsDirectory = Platform.environment['FLUTTER_TEST_OUTPUTS_DIR'] ?? 'build'; | ||
|
|
||
| String testOutputFilename = 'e2e_perf_summary'; | ||
|
|
||
| Future<void> main() async { | ||
| final FlutterDriver driver = await FlutterDriver.connect(); | ||
| final String jsonResult = | ||
| await driver.requestData(null, timeout: const Duration(minutes: 1)); | ||
| final e2e.Response response = e2e.Response.fromJson(jsonResult); | ||
| await driver.close(); | ||
|
|
||
| if (response.allTestsPassed) { | ||
| print('All tests passed.'); | ||
|
|
||
| await fs.directory(testOutputsDirectory).create(recursive: true); | ||
| final File file = fs.file(path.join( | ||
| testOutputsDirectory, | ||
| '$testOutputFilename.json' | ||
| )); | ||
| final String resultString = _encodeJson( | ||
| response.data['performance'] as Map<String, dynamic>, | ||
| true, | ||
| ); | ||
| await file.writeAsString(resultString); | ||
|
|
||
| exit(0); | ||
| } else { | ||
| print('Failure Details:\n${response.formattedFailureDetails}'); | ||
| exit(1); | ||
| } | ||
| } | ||
|
|
||
| String _encodeJson(Map<String, dynamic> jsonObject, bool pretty) { | ||
| return pretty | ||
| ? _prettyEncoder.convert(jsonObject) | ||
| : json.encode(jsonObject); | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| // Copyright 2014 The Flutter Authors. All rights reserved. | ||
| // Use of this source code is governed by a BSD-style license that can be | ||
| // found in the LICENSE file. | ||
|
|
||
| import 'dart:async'; | ||
|
|
||
| import 'package:flutter_devicelab/tasks/perf_tests.dart'; | ||
| import 'package:flutter_devicelab/framework/adb.dart'; | ||
| import 'package:flutter_devicelab/framework/framework.dart'; | ||
|
|
||
| Future<void> main() async { | ||
| deviceOperatingSystem = DeviceOperatingSystem.android; | ||
| await task(createCullOpacityPerfE2ETest()); | ||
| } |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This file probably belongs to
test_driveras this is unlikely to be used as a unit test.Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Same reason above. #61509 (comment)