[web] Allow tests to run with multiple renderers (flutter#37061)

Author: Harry Terkelsen

lib/web_ui/dev/steps/compile_tests_step.dart

@@ -194,37 +194,20 @@ Future<void> copyCanvasKitFiles({bool useLocalCanvasKit = false}) async {
 Future<void> compileTests(List<FilePath> testFiles) async {
   final Stopwatch stopwatch = Stopwatch()..start();
 
-  // Separate HTML targets from CanvasKit targets because the two use
-  // different dart2js options.
-  final List<FilePath> htmlTargets = <FilePath>[];
-  final List<FilePath> canvasKitTargets = <FilePath>[];
-  final List<FilePath> skwasmTargets = <FilePath>[];
-  final String canvasKitTestDirectory =
-      pathlib.join(environment.webUiTestDir.path, 'canvaskit');
-  final String skwasmTestDirectory =
-      pathlib.join(environment.webUiTestDir.path, 'skwasm');
-  for (final FilePath testFile in testFiles) {
-    if (pathlib.isWithin(canvasKitTestDirectory, testFile.absolute)) {
-      canvasKitTargets.add(testFile);
-    } else if (pathlib.isWithin(skwasmTestDirectory, testFile.absolute)) {
-      skwasmTargets.add(testFile);
-    } else {
-      htmlTargets.add(testFile);
-    }
-  }
+  final TestsByRenderer sortedTests = sortTestsByRenderer(testFiles);
 
   await Future.wait(<Future<void>>[
-    if (htmlTargets.isNotEmpty)
-      _compileTestsInParallel(targets: htmlTargets),
-    if (canvasKitTargets.isNotEmpty)
-      _compileTestsInParallel(targets: canvasKitTargets, forCanvasKit: true),
-    if (skwasmTargets.isNotEmpty)
-      _compileTestsInParallel(targets: skwasmTargets, forSkwasm: true),
+    if (sortedTests.htmlTests.isNotEmpty)
+      _compileTestsInParallel(targets: sortedTests.htmlTests),
+    if (sortedTests.canvasKitTests.isNotEmpty)
+      _compileTestsInParallel(targets: sortedTests.canvasKitTests, renderer: Renderer.canvasKit),
+    if (sortedTests.skwasmTests.isNotEmpty)
+      _compileTestsInParallel(targets: sortedTests.skwasmTests, renderer: Renderer.skwasm),
   ]);
 
   stopwatch.stop();
 
-  final int targetCount = htmlTargets.length + canvasKitTargets.length;
+  final int targetCount = sortedTests.numTargetsToCompile;
   print(
     'Built $targetCount tests in ${stopwatch.elapsedMilliseconds ~/ 1000} '
     'seconds using $_dart2jsConcurrency concurrent dart2js processes.',
@@ -239,12 +222,11 @@ final Pool _dart2jsPool = Pool(_dart2jsConcurrency);
 /// Spawns multiple dart2js processes to compile [targets] in parallel.
 Future<void> _compileTestsInParallel({
   required List<FilePath> targets,
-  bool forCanvasKit = false,
-  bool forSkwasm = false,
+  Renderer renderer = Renderer.html,
 }) async {
   final Stream<bool> results = _dart2jsPool.forEach(
     targets,
-    (FilePath file) => compileUnitTest(file, forCanvasKit: forCanvasKit, forSkwasm: forSkwasm),
+    (FilePath file) => compileUnitTest(file, renderer: renderer),
   );
   await for (final bool isSuccess in results) {
     if (!isSuccess) {
@@ -261,7 +243,7 @@ Future<void> _compileTestsInParallel({
 /// Dart2js creates the following outputs:
 /// - target.browser_test.dart.js
 /// - target.browser_test.dart.js.deps
-/// - target.browser_test.dart.js.maps
+/// - target.browser_test.dart.js.map
 /// under the same directory with test file. If all these files are not in
 /// the same directory, Chrome dev tools cannot load the source code during
 /// debug.
@@ -270,14 +252,18 @@ Future<void> _compileTestsInParallel({
 /// directory before test are build. See [_copyFilesFromTestToBuild].
 ///
 /// Later the extra files will be deleted in [_cleanupExtraFilesUnderTestDir].
-Future<bool> compileUnitTest(FilePath input, {required bool forCanvasKit, required bool forSkwasm}) async {
+Future<bool> compileUnitTest(FilePath input, {required Renderer renderer}) async {
+  // Compile to different directories for different renderers. This allows us
+  // to run the same test in multiple renderers.
   final String targetFileName = pathlib.join(
     environment.webUiBuildDir.path,
+    getBuildDirForRenderer(renderer),
     '${input.relativeToWebUi}.browser_test.dart.js',
   );
 
   final io.Directory directoryToTarget = io.Directory(pathlib.join(
       environment.webUiBuildDir.path,
+      getBuildDirForRenderer(renderer),
       pathlib.dirname(input.relativeToWebUi)));
 
   if (!directoryToTarget.existsSync()) {
@@ -295,8 +281,8 @@ Future<bool> compileUnitTest(FilePath input, {required bool forCanvasKit, requir
     // are designed to run in one specific mode. So instead, we specify the
     // renderer explicitly.
     '-DFLUTTER_WEB_AUTO_DETECT=false',
-    '-DFLUTTER_WEB_USE_SKIA=$forCanvasKit',
-    '-DFLUTTER_WEB_USE_SKWASM=$forSkwasm',
+    '-DFLUTTER_WEB_USE_SKIA=${renderer == Renderer.canvasKit}',
+    '-DFLUTTER_WEB_USE_SKWASM=${renderer == Renderer.skwasm}',
 
     '-O2',
     '-o',

lib/web_ui/dev/steps/run_tests_step.dart

@@ -68,15 +68,46 @@ class RunTestsStep implements PipelineStep {
     final SkiaGoldClient? skiaClient = await _createSkiaClient();
     final List<FilePath> testFiles = this.testFiles ?? findAllTests();
 
-    await _runTestBatch(
-      testFiles: testFiles,
-      browserEnvironment: browserEnvironment,
-      expectFailure: false,
-      isDebug: isDebug,
-      doUpdateScreenshotGoldens: doUpdateScreenshotGoldens,
-      skiaClient: skiaClient,
-      overridePathToCanvasKit: overridePathToCanvasKit,
-    );
+    final TestsByRenderer sortedTests = sortTestsByRenderer(testFiles);
+
+    if (sortedTests.htmlTests.isNotEmpty) {
+      await _runTestBatch(
+        testFiles: sortedTests.htmlTests,
+        renderer: Renderer.html,
+        browserEnvironment: browserEnvironment,
+        expectFailure: false,
+        isDebug: isDebug,
+        doUpdateScreenshotGoldens: doUpdateScreenshotGoldens,
+        skiaClient: skiaClient,
+        overridePathToCanvasKit: overridePathToCanvasKit,
+      );
+    }
+
+    if (sortedTests.canvasKitTests.isNotEmpty) {
+      await _runTestBatch(
+        testFiles: sortedTests.canvasKitTests,
+        renderer: Renderer.canvasKit,
+        browserEnvironment: browserEnvironment,
+        expectFailure: false,
+        isDebug: isDebug,
+        doUpdateScreenshotGoldens: doUpdateScreenshotGoldens,
+        skiaClient: skiaClient,
+        overridePathToCanvasKit: overridePathToCanvasKit,
+      );
+    }
+
+    if (sortedTests.skwasmTests.isNotEmpty) {
+      await _runTestBatch(
+        testFiles: sortedTests.skwasmTests,
+        renderer: Renderer.skwasm,
+        browserEnvironment: browserEnvironment,
+        expectFailure: false,
+        isDebug: isDebug,
+        doUpdateScreenshotGoldens: doUpdateScreenshotGoldens,
+        skiaClient: skiaClient,
+        overridePathToCanvasKit: overridePathToCanvasKit,
+      );
+    }
 
     await browserEnvironment.cleanup();
 
@@ -143,6 +174,7 @@ Future<void> _prepareTestResultsDirectory() async {
 /// value if any tests fail.
 Future<void> _runTestBatch({
   required List<FilePath> testFiles,
+  required Renderer renderer,
   required bool isDebug,
   required BrowserEnvironment browserEnvironment,
   required bool doUpdateScreenshotGoldens,
@@ -154,6 +186,10 @@ Future<void> _runTestBatch({
     environment.webUiRootDir.path,
     browserEnvironment.packageTestConfigurationYamlFile,
   );
+  final String precompiledBuildDir = pathlib.join(
+    environment.webUiBuildDir.path,
+    getBuildDirForRenderer(renderer),
+  );
   final List<String> testArgs = <String>[
     ...<String>['-r', 'compact'],
     // Disable concurrency. Running with concurrency proved to be flaky.
@@ -163,7 +199,7 @@ Future<void> _runTestBatch({
     if (expectFailure)
       '--reporter=name-only',
     '--platform=${browserEnvironment.packageTestRuntime.identifier}',
-    '--precompiled=${environment.webUiBuildDir.path}',
+    '--precompiled=$precompiledBuildDir',
     '--configuration=$configurationFilePath',
     '--',
     ...testFiles.map((FilePath f) => f.relativeToWebUi),
@@ -183,6 +219,7 @@ Future<void> _runTestBatch({
   ], () {
     return BrowserPlatform.start(
       browserEnvironment: browserEnvironment,
+      renderer: renderer,
       // It doesn't make sense to update a screenshot for a test that is
       // expected to fail.
       doUpdateScreenshotGoldens: !expectFailure && doUpdateScreenshotGoldens,

lib/web_ui/dev/test_platform.dart

@@ -38,12 +38,14 @@ import 'package:web_test_utils/image_compare.dart';
 
 import 'browser.dart';
 import 'environment.dart' as env;
+import 'utils.dart';
 
 /// Custom test platform that serves web engine unit tests.
 class BrowserPlatform extends PlatformPlugin {
   BrowserPlatform._({
     required this.browserEnvironment,
     required this.server,
+    required this.renderer,
     required this.isDebug,
     required this.doUpdateScreenshotGoldens,
     required this.packageConfig,
@@ -100,6 +102,7 @@ class BrowserPlatform extends PlatformPlugin {
   /// instead of failing the test on screenshot mismatches.
   static Future<BrowserPlatform> start({
     required BrowserEnvironment browserEnvironment,
+    required Renderer renderer,
     required bool doUpdateScreenshotGoldens,
     required SkiaGoldClient? skiaClient,
     required String? overridePathToCanvasKit,
@@ -108,6 +111,7 @@ class BrowserPlatform extends PlatformPlugin {
         shelf_io.IOServer(await HttpMultiServer.loopback(0));
     return BrowserPlatform._(
       browserEnvironment: browserEnvironment,
+      renderer: renderer,
       server: server,
       isDebug: Configuration.current.pauseAfterLoad,
       doUpdateScreenshotGoldens: doUpdateScreenshotGoldens,
@@ -128,6 +132,9 @@ class BrowserPlatform extends PlatformPlugin {
   /// Provides the environment for the browser running tests.
   final BrowserEnvironment browserEnvironment;
 
+  /// The renderer that tests are running under.
+  final Renderer renderer;
+
   /// The URL for this server.
   Uri get url => server.url.resolve('/');
 
@@ -384,11 +391,21 @@ class BrowserPlatform extends PlatformPlugin {
   ///
   /// This is used for trivial use-cases, such as `favicon.ico`, host pages, etc.
   shelf.Response buildDirectoryHandler(shelf.Request request) {
-    final File fileInBuild = File(p.join(
+    File fileInBuild = File(p.join(
       env.environment.webUiBuildDir.path,
       request.url.path,
     ));
 
+    // If we can't find the file in the top-level `build` directory, then it
+    // may be in the renderer-specific `build` subdirectory.
+    if (!fileInBuild.existsSync()) {
+      fileInBuild = File(p.join(
+        env.environment.webUiBuildDir.path,
+        getBuildDirForRenderer(renderer),
+        request.url.path,
+      ));
+    }
+
     if (!fileInBuild.existsSync()) {
       return shelf.Response.notFound('File not found: ${request.url.path}');
     }
@@ -517,6 +534,7 @@ class BrowserPlatform extends PlatformPlugin {
       future: completer.future,
       packageConfig: packageConfig,
       debug: isDebug,
+      renderer: renderer,
     );
 
     // Store null values for browsers that error out so we know not to load them
@@ -612,7 +630,7 @@ class BrowserManager {
   /// Creates a new BrowserManager that communicates with the browser over
   /// [webSocket].
   BrowserManager._(this.packageConfig, this._browser, this._browserEnvironment,
-      WebSocketChannel webSocket) {
+      this._renderer, WebSocketChannel webSocket) {
     // The duration should be short enough that the debugging console is open as
     // soon as the user is done setting breakpoints, but long enough that a test
     // doing a lot of synchronous work doesn't trigger a false positive.
@@ -658,6 +676,9 @@ class BrowserManager {
   /// The browser environment for this test.
   final BrowserEnvironment _browserEnvironment;
 
+  /// The renderer for this test.
+  final Renderer _renderer;
+
   /// The channel used to communicate with the browser.
   ///
   /// This is connected to a page running `static/host.dart`.
@@ -722,6 +743,7 @@ class BrowserManager {
     required Uri url,
     required Future<WebSocketChannel> future,
     required PackageConfig packageConfig,
+    required Renderer renderer,
     bool debug = false,
   }) async {
     final Browser browser =
@@ -732,6 +754,7 @@ class BrowserManager {
         future: future,
         packageConfig: packageConfig,
         browser: browser,
+        renderer: renderer,
         debug: debug);
   }
 
@@ -741,6 +764,7 @@ class BrowserManager {
     required Future<WebSocketChannel> future,
     required PackageConfig packageConfig,
     required Browser browser,
+    required Renderer renderer,
     bool debug = false,
   }) {
     final Completer<BrowserManager> completer = Completer<BrowserManager>();
@@ -762,7 +786,7 @@ class BrowserManager {
         return;
       }
       completer.complete(BrowserManager._(
-          packageConfig, browser, browserEnvironment, webSocket));
+          packageConfig, browser, browserEnvironment, renderer, webSocket));
     }).catchError((Object error, StackTrace stackTrace) {
       browser.close();
       if (completer.isCompleted) {
@@ -846,7 +870,7 @@ class BrowserManager {
         final String pathToTest = p.dirname(path);
 
         final String mapPath = p.join(env.environment.webUiRootDir.path,
-            'build', pathToTest, sourceMapFileName);
+            'build', getBuildDirForRenderer(_renderer), pathToTest, sourceMapFileName);
 
         final Map<String, Uri> packageMap = <String, Uri>{
           for (Package p in packageConfig.packages) p.name: p.packageUriRoot

lib/web_ui/dev/utils.dart

@@ -379,3 +379,68 @@ List<FilePath> findAllTests() {
           path.relative(f.path, from: environment.webUiRootDir.path)))
       .toList();
 }
+
+/// The renderer used to run the test.
+enum Renderer {
+  html,
+  canvasKit,
+  skwasm,
+}
+
+/// The `FilePath`s for all the tests, organized by renderer.
+class TestsByRenderer {
+  TestsByRenderer(this.htmlTests, this.canvasKitTests, this.skwasmTests);
+
+  /// Tests which should be run with the HTML renderer.
+  final List<FilePath> htmlTests;
+
+  /// Tests which should be run with the CanvasKit renderer.
+  final List<FilePath> canvasKitTests;
+
+  /// Tests which should be run with the Skwasm renderer.
+  final List<FilePath> skwasmTests;
+
+  /// The total number of targets to compile.
+  ///
+  /// The number of uiTests is doubled since they are compiled twice: once for
+  /// the HTML renderer and once for the CanvasKit renderer.
+  int get numTargetsToCompile => htmlTests.length + canvasKitTests.length + skwasmTests.length;
+}
+
+/// Given a list of test files, organizes them by which renderer should run them.
+TestsByRenderer sortTestsByRenderer(List<FilePath> testFiles) {
+  final List<FilePath> htmlTargets = <FilePath>[];
+  final List<FilePath> canvasKitTargets = <FilePath>[];
+  final List<FilePath> skwasmTargets = <FilePath>[];
+  final String canvasKitTestDirectory =
+      path.join(environment.webUiTestDir.path, 'canvaskit');
+  final String skwasmTestDirectory =
+      path.join(environment.webUiTestDir.path, 'skwasm');
+  final String uiTestDirectory =
+      path.join(environment.webUiTestDir.path, 'ui');
+  for (final FilePath testFile in testFiles) {
+    if (path.isWithin(canvasKitTestDirectory, testFile.absolute)) {
+      canvasKitTargets.add(testFile);
+    } else if (path.isWithin(skwasmTestDirectory, testFile.absolute)) {
+      skwasmTargets.add(testFile);
+    } else if (path.isWithin(uiTestDirectory, testFile.absolute)) {
+      htmlTargets.add(testFile);
+      canvasKitTargets.add(testFile);
+    } else {
+      htmlTargets.add(testFile);
+    }
+  }
+  return TestsByRenderer(htmlTargets, canvasKitTargets, skwasmTargets);
+}
+
+/// The build directory to compile a test into given the renderer.
+String getBuildDirForRenderer(Renderer renderer) {
+  switch (renderer) {
+    case Renderer.html:
+      return 'html_tests';
+    case Renderer.canvasKit:
+      return 'canvaskit_tests';
+    case Renderer.skwasm:
+      return 'skwasm_tests';
+  }
+}