|
2 | 2 | // Use of this source code is governed by a BSD-style license that can be |
3 | 3 | // found in the LICENSE file. |
4 | 4 |
|
5 | | -import 'dart:convert' show json; |
6 | | -import 'dart:io' show sleep, stdout; |
| 5 | +import 'dart:convert' show JsonEncoder, json; |
| 6 | +import 'dart:io' show File, sleep, stdout; |
7 | 7 |
|
8 | 8 | import 'package:flutter_driver/flutter_driver.dart'; |
9 | 9 | import 'package:test/test.dart' hide TypeMatcher, isInstanceOf; |
@@ -220,6 +220,94 @@ Future<void> runDemos( |
220 | 220 | } |
221 | 221 | } |
222 | 222 |
|
| 223 | +/// Summarizes how well the Impeller pipelines created up front (warmed) covered |
| 224 | +/// the draws made so far, as reported by `_flutter.getPipelineUsage`. |
| 225 | +/// |
| 226 | +/// Returns no metrics, rather than failing the benchmark, if pipeline usage is |
| 227 | +/// unavailable, for example because the app is not rendering with Impeller. |
| 228 | +Future<Map<String, Object?>> pipelineWarmupMetrics(FlutterDriver driver) async { |
| 229 | + try { |
| 230 | + final Map<String, Object?>? views = (await driver.serviceClient.callMethod( |
| 231 | + '_flutter.listViews', |
| 232 | + )).json; |
| 233 | + if (views case {'views': [{'id': final String viewId}, ...]}) { |
| 234 | + final Map<String, Object?>? usage = (await driver.serviceClient.callMethod( |
| 235 | + '_flutter.getPipelineUsage', |
| 236 | + args: <String, Object?>{'viewId': viewId}, |
| 237 | + )).json; |
| 238 | + if (usage case {'pipelines': final List<Object?> pipelines}) { |
| 239 | + return _summarizePipelineUsage(pipelines); |
| 240 | + } |
| 241 | + throw FormatException( |
| 242 | + 'Expected a "pipelines" list in the _flutter.getPipelineUsage response for view ' |
| 243 | + '$viewId, but got keys ${usage?.keys.toList()}.', |
| 244 | + ); |
| 245 | + } |
| 246 | + throw FormatException( |
| 247 | + 'Expected at least one Flutter view in the _flutter.listViews response, but got $views.', |
| 248 | + ); |
| 249 | + } catch (error) { |
| 250 | + stdout.writeln('Not reporting pipeline warmup metrics: $error'); |
| 251 | + return const <String, Object?>{}; |
| 252 | + } |
| 253 | +} |
| 254 | + |
| 255 | +/// The warmup metrics for the `pipelines` of a `_flutter.getPipelineUsage` |
| 256 | +/// response. Also logs the pipelines that were created lazily, to help track |
| 257 | +/// down regressions. |
| 258 | +Map<String, Object?> _summarizePipelineUsage(List<Object?> pipelines) { |
| 259 | + var totalDraws = 0; |
| 260 | + var warmedDraws = 0; |
| 261 | + var warmedNeverDrawn = 0; |
| 262 | + final lazyPipelines = <String>[]; |
| 263 | + for (final pipeline in pipelines) { |
| 264 | + if (pipeline case { |
| 265 | + 'label': final String label, |
| 266 | + 'constants': final List<Object?> constants, |
| 267 | + 'count': final int count, |
| 268 | + 'eagerly_created': final bool eagerlyCreated, |
| 269 | + }) { |
| 270 | + totalDraws += count; |
| 271 | + if (!eagerlyCreated) { |
| 272 | + final name = constants.isEmpty ? label : '$label $constants'; |
| 273 | + lazyPipelines.add('$name: $count draws'); |
| 274 | + } else if (count == 0) { |
| 275 | + warmedNeverDrawn += 1; |
| 276 | + } else { |
| 277 | + warmedDraws += count; |
| 278 | + } |
| 279 | + } else { |
| 280 | + throw FormatException( |
| 281 | + 'Expected a label, constants, count and eagerly_created in each ' |
| 282 | + '_flutter.getPipelineUsage entry, but got $pipeline.', |
| 283 | + ); |
| 284 | + } |
| 285 | + } |
| 286 | + if (totalDraws == 0) { |
| 287 | + throw FormatException( |
| 288 | + '_flutter.getPipelineUsage reported ${pipelines.length} pipelines, but no draws.', |
| 289 | + ); |
| 290 | + } |
| 291 | + |
| 292 | + final double warmedDrawPercent = double.parse( |
| 293 | + (warmedDraws * 100 / totalDraws).toStringAsFixed(2), |
| 294 | + ); |
| 295 | + stdout.writeln( |
| 296 | + 'Impeller pipeline warmup: ${warmedDrawPercent.toStringAsFixed(2)}% of $totalDraws draws ' |
| 297 | + 'used a warmed pipeline, $warmedNeverDrawn warmed pipelines were never drawn with, and ' |
| 298 | + '${lazyPipelines.length} pipelines were created lazily:', |
| 299 | + ); |
| 300 | + lazyPipelines.sort(); |
| 301 | + for (final pipeline in lazyPipelines) { |
| 302 | + stdout.writeln(' $pipeline'); |
| 303 | + } |
| 304 | + return <String, Object?>{ |
| 305 | + 'shader_warmed_draw_percent': warmedDrawPercent, |
| 306 | + 'shader_lazy_variants': lazyPipelines.length, |
| 307 | + 'shader_warmed_never_drawn_variants': warmedNeverDrawn, |
| 308 | + }; |
| 309 | +} |
| 310 | + |
223 | 311 | void main([List<String> args = const <String>[]]) { |
224 | 312 | group('Flutter Gallery transitions', () { |
225 | 313 | late FlutterDriver driver; |
@@ -335,11 +423,20 @@ void main([List<String> args = const <String>[]]) { |
335 | 423 | ); |
336 | 424 |
|
337 | 425 | final summary = TimelineSummary.summarize(timeline); |
338 | | - await summary.writeTimelineToFile('transitions', pretty: true); |
| 426 | + // The summary is written once every demo has run, so that it can include |
| 427 | + // the pipeline warmup metrics. |
| 428 | + await summary.writeTimelineToFile('transitions', pretty: true, includeSummary: false); |
339 | 429 |
|
340 | 430 | // Execute the remaining tests. |
341 | 431 | final unprofiledDemos = Set<String>.from(_allDemos)..removeAll(_profiledDemos); |
342 | 432 | await runDemos(unprofiledDemos.toList(), driver); |
| 433 | + |
| 434 | + final summaryJson = <String, Object?>{ |
| 435 | + ...summary.summaryJson, |
| 436 | + ...await pipelineWarmupMetrics(driver), |
| 437 | + }; |
| 438 | + await File('$testOutputsDirectory/transitions.timeline_summary.json') |
| 439 | + .writeAsString(const JsonEncoder.withIndent(' ').convert(summaryJson)); |
343 | 440 | }, timeout: Timeout.none); |
344 | 441 | }); |
345 | 442 | } |
0 commit comments