Skip to content

Commit c778297

Browse files
[web] Support local screenshot and golden testing in felt test (flutter#191318)
Enables local screenshot testing in felt test by downloading baselines from Skia Gold, performing local pixel diffing (both precise and fuzzy), and caching baselines on disk. Fixes flutter#191314 Fixes flutter#63710 Part of flutter#191313 ## Pre-launch Checklist - [x] I read the [Contributor Guide] and followed the process outlined there for submitting PRs. - [x] I read the [AI contribution guidelines] and understand my responsibilities, or I am not using AI tools. - [x] I read the [Tree Hygiene] wiki page, which explains my responsibilities. - [x] I read and followed the [Flutter Style Guide], including [Features we expect every widget to implement]. - [x] I signed the [CLA]. - [x] I listed at least one issue that this PR fixes in the description above. - [x] I updated/added relevant documentation (doc comments with `///`). - [x] I added new tests to check the change I am making, or this PR is [test-exempt]. - [x] I followed the [breaking change policy] and added [Data Driven Fixes] where supported. - [x] All existing and new tests are passing. If you need help, consider asking for advice on the #hackers-new channel on [Discord]. If this change needs to override an active code freeze, provide a comment explaining why. The code freeze workflow can be overridden by code reviewers. See pinned issues for any active code freezes with guidance. **Note**: The Flutter team is currently trialing the use of [Gemini Code Assist for GitHub](https://developers.google.com/gemini-code-assist/docs/review-github-code). Comments from the `gemini-code-assist` bot should not be taken as authoritative feedback from the Flutter team. If you find its comments useful you can update your code accordingly, but if you are unsure or disagree with the feedback, please feel free to wait for a Flutter team member's review for guidance on which automated comments should be addressed. <!-- Links --> [Contributor Guide]: https://github.com/flutter/flutter/blob/main/docs/contributing/Tree-hygiene.md#overview [AI contribution guidelines]: https://github.com/flutter/flutter/blob/main/docs/contributing/Tree-hygiene.md#ai-contribution-guidelines [Tree Hygiene]: https://github.com/flutter/flutter/blob/main/docs/contributing/Tree-hygiene.md [test-exempt]: https://github.com/flutter/flutter/blob/main/docs/contributing/Tree-hygiene.md#tests [Flutter Style Guide]: https://github.com/flutter/flutter/blob/main/docs/contributing/Style-guide-for-Flutter-repo.md [Features we expect every widget to implement]: https://github.com/flutter/flutter/blob/main/docs/contributing/Style-guide-for-Flutter-repo.md#features-we-expect-every-widget-to-implement [CLA]: https://cla.developers.google.com/ [flutter/tests]: https://github.com/flutter/tests [breaking change policy]: https://github.com/flutter/flutter/blob/main/docs/contributing/Tree-hygiene.md#handling-breaking-changes [Discord]: https://github.com/flutter/flutter/blob/main/docs/contributing/Chat.md [Data Driven Fixes]: https://github.com/flutter/flutter/blob/main/docs/contributing/Data-driven-Fixes.md --------- Co-authored-by: Mouad Debbar <[email protected]>
1 parent 4f4467b commit c778297

11 files changed

Lines changed: 1087 additions & 121 deletions

File tree

‎engine/src/flutter/lib/web_ui/dev/environment.dart‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,9 @@ class Environment {
8383
/// Path to the engine's "src" directory.
8484
final io.Directory engineSrcDir;
8585

86+
/// Path to the root flutter directory (which itself contains `engine/src/flutter`).
87+
final io.Directory flutterRootDir;
88+
8689
/// Path to the engine's "tools" directory.
8790
final io.Directory engineToolsDir;
8891

@@ -132,11 +135,10 @@ class Environment {
132135
'dart2wasm.dart',
133136
);
134137

135-
/// Path to the root flutter directory (which itself contains `engine/src/flutter`).
136-
final io.Directory flutterRootDir;
137-
138138
/// Path to where github.com/flutter/engine is checked out inside the engine workspace.
139139
io.Directory get flutterDirectory => io.Directory(pathlib.join(engineSrcDir.path, 'flutter'));
140+
141+
/// Path to the engine's "web_sdk" directory.
140142
io.Directory get webSdkRootDir => io.Directory(pathlib.join(flutterDirectory.path, 'web_sdk'));
141143

142144
/// Path to the "web_engine_tester" package.
@@ -148,6 +150,7 @@ class Environment {
148150
/// This is where compiled test output goes.
149151
io.Directory get webUiBuildDir => io.Directory(pathlib.join(outDir.path, 'web_tests'));
150152

153+
/// Path to artifacts generated for web tests.
151154
io.Directory get webTestsArtifactsDir =>
152155
io.Directory(pathlib.join(webUiBuildDir.path, 'artifacts'));
153156

@@ -157,7 +160,7 @@ class Environment {
157160
/// Path to the ".dart_tool" directory living under `engine/src/flutter`.
158161
///
159162
/// This is a designated area for tool downloads which can be used by
160-
/// multiple platforms. For exampe: Flutter repo for e2e tests.
163+
/// multiple platforms. For example: Flutter repo for e2e tests.
161164
io.Directory get engineDartToolDir =>
162165
io.Directory(pathlib.join(engineSrcDir.path, 'flutter', '.dart_tool'));
163166

@@ -179,4 +182,8 @@ class Environment {
179182
/// bucket by LUCI.
180183
io.Directory get webUiTestResultsDirectory =>
181184
io.Directory(pathlib.join(webUiDartToolDir.path, 'test_results'));
185+
186+
/// Path to the directory for caching web goldens baselines and comparison failures.
187+
io.Directory get webUiGoldensCacheDirectory =>
188+
io.Directory(pathlib.join(webUiDartToolDir.path, 'web_goldens_cache'));
182189
}

‎engine/src/flutter/lib/web_ui/dev/steps/copy_artifacts_step.dart‎

Lines changed: 47 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ class CopyArtifactsStep implements PipelineStep {
5959
'storage.googleapis.com',
6060
'${realmComponent}flutter_infra_release/flutter/${realm == LuciRealm.Try ? gitRevision : contentHash}/flutter-web-sdk.zip',
6161
);
62-
final http.Response response = await http.Client().get(url);
62+
final http.Response response = await http.get(url);
6363
if (response.statusCode != 200) {
6464
throw ToolExit(
6565
'Could not download flutter-web-sdk.zip from cloud bucket at URL: $url. Response status code: ${response.statusCode}',
@@ -196,13 +196,16 @@ class CopyArtifactsStep implements PipelineStep {
196196
'assets',
197197
'fallback_fonts',
198198
);
199-
for (final io.File file in fallbackFontsSource.listSync(recursive: true).whereType<io.File>()) {
200-
final String relativePath = pathlib.relative(file.path, from: fallbackFontsSource.path);
201-
final destinationFile = io.File(pathlib.join(fallbackFontsDestinationPath, relativePath));
202-
if (!destinationFile.parent.existsSync()) {
203-
destinationFile.parent.createSync(recursive: true);
199+
if (fallbackFontsSource.existsSync()) {
200+
for (final io.File file
201+
in fallbackFontsSource.listSync(recursive: true).whereType<io.File>()) {
202+
final String relativePath = pathlib.relative(file.path, from: fallbackFontsSource.path);
203+
final destinationFile = io.File(pathlib.join(fallbackFontsDestinationPath, relativePath));
204+
if (!destinationFile.parent.existsSync()) {
205+
destinationFile.parent.createSync(recursive: true);
206+
}
207+
file.copySync(destinationFile.path);
204208
}
205-
file.copySync(destinationFile.path);
206209
}
207210
}
208211

@@ -218,43 +221,53 @@ class CopyArtifactsStep implements PipelineStep {
218221
),
219222
);
220223

221-
for (final io.File imageFile in testImagesDir.listSync(recursive: true).whereType<io.File>()) {
222-
// Skip files that are used by Skia to test handling of invalid input.
223-
final String imageBaseName = pathlib.basename(imageFile.path);
224-
if (imageBaseName.contains('invalid') || imageBaseName.contains('missing_eof')) {
225-
continue;
224+
if (testImagesDir.existsSync()) {
225+
for (final io.File imageFile
226+
in testImagesDir.listSync(recursive: true).whereType<io.File>()) {
227+
// Skip files that are used by Skia to test handling of invalid input.
228+
final String imageBaseName = pathlib.basename(imageFile.path);
229+
if (imageBaseName.contains('invalid') || imageBaseName.contains('missing_eof')) {
230+
continue;
231+
}
232+
final destination = io.File(
233+
pathlib.join(
234+
environment.webTestsArtifactsDir.path,
235+
'test_images',
236+
pathlib.relative(imageFile.path, from: testImagesDir.path),
237+
),
238+
);
239+
destination.createSync(recursive: true);
240+
await imageFile.copy(destination.path);
226241
}
227-
final destination = io.File(
228-
pathlib.join(
229-
environment.webTestsArtifactsDir.path,
230-
'test_images',
231-
pathlib.relative(imageFile.path, from: testImagesDir.path),
232-
),
233-
);
234-
destination.createSync(recursive: true);
235-
await imageFile.copy(destination.path);
236242
}
237243
}
238244

239245
Future<void> copyFlutterJsFiles(String sourcePath) async {
240-
final flutterJsInputDirectory = io.Directory(sourcePath);
246+
var flutterJsInputDirectory = io.Directory(sourcePath);
247+
if (!flutterJsInputDirectory.existsSync()) {
248+
flutterJsInputDirectory = io.Directory(
249+
pathlib.join(environment.webUiRootDir.path, 'flutter_js', 'src'),
250+
);
251+
}
241252
final String targetDirectoryPath = pathlib.join(
242253
environment.webTestsArtifactsDir.path,
243254
'flutter_js',
244255
);
245256

246-
for (final io.File sourceFile
247-
in flutterJsInputDirectory.listSync(recursive: true).whereType<io.File>()) {
248-
final String relativePath = pathlib.relative(
249-
sourceFile.path,
250-
from: flutterJsInputDirectory.path,
251-
);
252-
final String targetPath = pathlib.join(targetDirectoryPath, relativePath);
253-
final targetFile = io.File(targetPath);
254-
if (!targetFile.parent.existsSync()) {
255-
targetFile.parent.createSync(recursive: true);
257+
if (flutterJsInputDirectory.existsSync()) {
258+
for (final io.File sourceFile
259+
in flutterJsInputDirectory.listSync(recursive: true).whereType<io.File>()) {
260+
final String relativePath = pathlib.relative(
261+
sourceFile.path,
262+
from: flutterJsInputDirectory.path,
263+
);
264+
final String targetPath = pathlib.join(targetDirectoryPath, relativePath);
265+
final targetFile = io.File(targetPath);
266+
if (!targetFile.parent.existsSync()) {
267+
targetFile.parent.createSync(recursive: true);
268+
}
269+
sourceFile.copySync(targetPath);
256270
}
257-
sourceFile.copySync(targetPath);
258271
}
259272
}
260273

@@ -281,9 +294,7 @@ class CopyArtifactsStep implements PipelineStep {
281294
// they are optional.
282295
continue;
283296
}
284-
{
285-
throw ToolExit('Built artifact not found at path "$sourceFile".');
286-
}
297+
throw ToolExit('Built artifact not found at path "$sourceFile".');
287298
}
288299
await targetFile.create(recursive: true);
289300
await sourceFile.copy(targetFile.path);

‎engine/src/flutter/lib/web_ui/dev/steps/run_suite_step.dart‎

Lines changed: 46 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -37,19 +37,40 @@ class RunSuiteStep implements PipelineStep {
3737
required this.overridePathToCanvasKit,
3838
required this.useDwarf,
3939
this.testFiles,
40+
this.isOffline = false,
41+
this.refreshGoldens = false,
4042
});
4143

44+
/// The test suite to run.
4245
final TestSuite suite;
43-
final Set<FilePath>? testFiles;
46+
47+
/// Whether to start the test in paused mode.
4448
final bool startPaused;
49+
50+
/// Whether to output verbose diagnostic logs.
4551
final bool isVerbose;
52+
53+
/// Whether to update screenshots in golden directory instead of comparing.
4654
final bool doUpdateScreenshotGoldens;
47-
final String? overridePathToCanvasKit;
48-
final bool useDwarf;
4955

5056
/// Require Skia Gold to be available and reachable.
5157
final bool requireSkiaGold;
5258

59+
/// An optional override path to CanvasKit artifacts.
60+
final String? overridePathToCanvasKit;
61+
62+
/// Whether to use DWARF stack traces.
63+
final bool useDwarf;
64+
65+
/// Specific test files to run within the suite.
66+
final Set<FilePath>? testFiles;
67+
68+
/// Whether to run tests offline using cached baselines.
69+
final bool isOffline;
70+
71+
/// Whether to force refresh cached goldens from Skia Gold.
72+
final bool refreshGoldens;
73+
5374
@override
5475
String get description => 'run_suite';
5576

@@ -99,6 +120,8 @@ class RunSuiteStep implements PipelineStep {
99120
skiaClient: skiaClient,
100121
overridePathToCanvasKit: overridePathToCanvasKit,
101122
isVerbose: isVerbose,
123+
isOffline: isOffline,
124+
refreshGoldens: refreshGoldens,
102125
);
103126
});
104127

@@ -204,6 +227,11 @@ class RunSuiteStep implements PipelineStep {
204227
final bool isWebParagraph = suite.runConfig.enableWebParagraph;
205228

206229
final dimensions = <String, String>{
230+
// All Flutter Web Engine screenshot tryjobs run on Linux on CI (LUCI).
231+
// Hardcoding 'Platform': 'linux' ensures that local test runs on macOS
232+
// and Windows query and match the official Linux CI golden baselines
233+
// stored in Skia Gold.
234+
'Platform': 'linux',
207235
'Browser': suite.runConfig.browser.name,
208236
if (isWasm) 'Wasm': 'true',
209237
'Renderer': rendererName,
@@ -228,25 +256,34 @@ class RunSuiteStep implements PipelineStep {
228256

229257
/// Checks whether the Skia Client is usable in this environment.
230258
Future<(bool, String?)> _checkSkiaClient(SkiaGoldClient skiaClient) async {
259+
// When offline, we don't attempt network connectivity to Skia Gold, but we still
260+
// return true so SkiaGoldClient is instantiated for local offline baseline comparisons.
261+
if (isOffline) {
262+
return (true, null);
263+
}
231264
// Now let's check whether Skia Gold is reachable or not.
232265
if (isLuci) {
233266
if (SkiaGoldClient.isAvailable()) {
234267
try {
235268
await skiaClient.auth();
236269
return (true, null);
237-
} catch (e) {
238-
print(e);
270+
} catch (error) {
271+
print(error);
239272
}
240273
}
241274
} else {
242275
try {
243276
// Check if we can reach Gold.
244277
await skiaClient.getExpectationForTest('');
245278
return (true, null);
246-
} on io.OSError catch (_) {
247-
return (false, 'OSError occurred, could not reach Gold');
248-
} on io.SocketException catch (_) {
249-
return (false, 'SocketException occurred, could not reach Gold');
279+
} on io.OSError catch (error) {
280+
return (false, 'OSError occurred, could not reach Gold: $error');
281+
} on io.SocketException catch (error) {
282+
return (false, 'SocketException occurred, could not reach Gold: $error');
283+
} on io.IOException catch (error) {
284+
return (false, 'Network error occurred, could not reach Gold: $error');
285+
} on FormatException catch (error) {
286+
return (false, 'Format error occurred, could not parse Gold response: $error');
250287
}
251288
}
252289

0 commit comments

Comments
 (0)