From 1606fa26edfaaf9d5b191137845143ec37b5adb8 Mon Sep 17 00:00:00 2001 From: Yegor Jbanov Date: Thu, 12 Dec 2024 16:46:27 -0800 Subject: [PATCH 1/5] [monorepo] split out build-only web engine into its own builder --- .ci.yaml | 26 ++++- ci/builders/linux_web_engine.json | 11 -- ci/builders/linux_web_engine_build.json | 41 ++++++++ lib/web_ui/dev/generate_builder_json.dart | 121 ++++++++++++++-------- 4 files changed, 143 insertions(+), 56 deletions(-) create mode 100644 ci/builders/linux_web_engine_build.json diff --git a/.ci.yaml b/.ci.yaml index 1687b69e18963..74449eb6a538a 100644 --- a/.ci.yaml +++ b/.ci.yaml @@ -380,11 +380,35 @@ targets: config_name: linux_license clobber: "true" - - name: Linux linux_web_engine + - name: Linux linux_web_engine_build recipe: engine_v2/engine_v2 timeout: 120 properties: release_build: "true" + config_name: linux_web_engine_build + # Do not remove(https://github.com/flutter/flutter/issues/144644) + # Scheduler will fail to get the platform + drone_dimensions: + - os=Linux + dimensions: + # This is needed so that orchestrators that only spawn subbuilds are not + # assigned to the large 32 core workers when doing release builds. + # For more details see the issue + # at https://github.com/flutter/flutter/issues/152186. + cores: "8" + runIf: + - DEPS + - .ci.yaml + - lib/web_ui/** + - web_sdk/** + - tools/** + - ci/** + - flutter_frontend_server/** + + - name: Linux linux_web_engine + recipe: engine_v2/engine_v2 + timeout: 120 + properties: config_name: linux_web_engine # Do not remove(https://github.com/flutter/flutter/issues/144644) # Scheduler will fail to get the platform diff --git a/ci/builders/linux_web_engine.json b/ci/builders/linux_web_engine.json index f4159952eafc2..ce0950e9c5e87 100644 --- a/ci/builders/linux_web_engine.json +++ b/ci/builders/linux_web_engine.json @@ -25,17 +25,6 @@ "flutter/web_sdk:flutter_web_sdk_archive" ] }, - "archives": [ - { - "name": "wasm_release", - "base_path": "out/wasm_release/zip_archives/", - "type": "gcs", - "include_paths": [ - "out/wasm_release/zip_archives/flutter-web-sdk.zip" - ], - "realm": "production" - } - ], "generators": { "tasks": [ { diff --git a/ci/builders/linux_web_engine_build.json b/ci/builders/linux_web_engine_build.json new file mode 100644 index 0000000000000..9bc499584aa50 --- /dev/null +++ b/ci/builders/linux_web_engine_build.json @@ -0,0 +1,41 @@ +{ + "_comment": "THIS IS A GENERATED FILE. Do not edit this file directly.", + "_comment2": "See `generate_builder_json.dart` for the generator code", + "builds": [ + { + "name": "web_build/artifacts", + "drone_dimensions": [ + "device_type=none", + "os=Linux", + "cores=32" + ], + "gclient_variables": { + "download_android_deps": false, + "download_jdk": false, + "download_emsdk": true + }, + "gn": [ + "--web", + "--runtime-mode=release", + "--no-goma" + ], + "ninja": { + "config": "wasm_release", + "targets": [ + "flutter/web_sdk:flutter_web_sdk_archive" + ] + }, + "archives": [ + { + "name": "wasm_release", + "base_path": "out/wasm_release/zip_archives/", + "type": "gcs", + "include_paths": [ + "out/wasm_release/zip_archives/flutter-web-sdk.zip" + ], + "realm": "production" + } + ] + } + ] +} diff --git a/lib/web_ui/dev/generate_builder_json.dart b/lib/web_ui/dev/generate_builder_json.dart index d4263e423b357..93e244550d401 100644 --- a/lib/web_ui/dev/generate_builder_json.dart +++ b/lib/web_ui/dev/generate_builder_json.dart @@ -28,34 +28,66 @@ class GenerateBuilderJsonCommand extends Command { final FeltConfig config = FeltConfig.fromFile( path.join(environment.webUiTestDir.path, 'felt_config.yaml') ); - final String configString = generate(config, packageLock); - final io.File configFile = io.File(path.join( + + // Generate the config that only builds the engine, but does not run tests. + // This allows starting framework as soon as the engine is built, without + // waiting for the engine tests. This is also used to skip tests entirely + // when running in the merge queue. + final String buildConfigString = generateBuildConfig(config, packageLock); + final io.File buildConfigFile = io.File(path.join( + environment.flutterDirectory.path, + 'ci', + 'builders', + 'linux_web_engine_build.json', + )); + buildConfigFile.writeAsStringSync('$buildConfigString\n'); + + // Generate the config for the full build that includes both the engine + // build and tests. + final String buildAndTestConfigString = generate(config, packageLock); + final io.File buildAndTestConfigFile = io.File(path.join( environment.flutterDirectory.path, 'ci', 'builders', 'linux_web_engine.json', )); - configFile.writeAsStringSync('$configString\n'); + buildAndTestConfigFile.writeAsStringSync('$buildAndTestConfigString\n'); + return true; } + String generateBuildConfig(FeltConfig config, PackageLock packageLock) { + final Map outputJson = { + '_comment': 'THIS IS A GENERATED FILE. Do not edit this file directly.', + '_comment2': 'See `generate_builder_json.dart` for the generator code', + 'builds': [ + _getArtifactBuildStep(name: 'web_build/artifacts', forTesting: false), + ], + }; + return const JsonEncoder.withIndent(' ').convert(outputJson); + } + String generate(FeltConfig config, PackageLock packageLock) { final Map outputJson = { '_comment': 'THIS IS A GENERATED FILE. Do not edit this file directly.', '_comment2': 'See `generate_builder_json.dart` for the generator code', 'builds': [ - _getArtifactBuildStep(), - for (final TestBundle bundle in config.testBundles) - _getBundleBuildStep(bundle), + _getArtifactBuildStep(name: 'web_tests/artifacts', forTesting: true), + if (true) + for (final TestBundle bundle in config.testBundles) + _getBundleBuildStep(bundle), ], 'tests': _getAllTestSteps(config.testSuites, packageLock) }; return const JsonEncoder.withIndent(' ').convert(outputJson); } - Map _getArtifactBuildStep() { + Map _getArtifactBuildStep({ + required String name, + required bool forTesting, + }) { return { - 'name': 'web_tests/artifacts', + 'name': name, 'drone_dimensions': [ 'device_type=none', 'os=Linux', @@ -77,44 +109,45 @@ class GenerateBuilderJsonCommand extends Command { 'flutter/web_sdk:flutter_web_sdk_archive' ] }, - 'archives': [ - { - 'name': 'wasm_release', - 'base_path': 'out/wasm_release/zip_archives/', - 'type': 'gcs', - 'include_paths': [ - 'out/wasm_release/zip_archives/flutter-web-sdk.zip' - ], - 'realm': 'production', - } - ], - 'generators': { - 'tasks': [ + if (!forTesting) + 'archives': [ { - 'name': 'check licenses', - 'parameters': [ - 'check-licenses' - ], - 'scripts': [ 'flutter/lib/web_ui/dev/felt' ], - - }, - { - 'name': 'web engine analysis', - 'parameters': [ - 'analyze' - ], - 'scripts': [ 'flutter/lib/web_ui/dev/felt' ], - }, - { - 'name': 'copy artifacts for web tests', - 'parameters': [ - 'test', - '--copy-artifacts', + 'name': 'wasm_release', + 'base_path': 'out/wasm_release/zip_archives/', + 'type': 'gcs', + 'include_paths': [ + 'out/wasm_release/zip_archives/flutter-web-sdk.zip' ], - 'scripts': [ 'flutter/lib/web_ui/dev/felt' ], - }, - ] - }, + 'realm': 'production', + } + ], + if (forTesting) + 'generators': { + 'tasks': [ + { + 'name': 'check licenses', + 'parameters': [ + 'check-licenses' + ], + 'scripts': [ 'flutter/lib/web_ui/dev/felt' ], + }, + { + 'name': 'web engine analysis', + 'parameters': [ + 'analyze' + ], + 'scripts': [ 'flutter/lib/web_ui/dev/felt' ], + }, + { + 'name': 'copy artifacts for web tests', + 'parameters': [ + 'test', + '--copy-artifacts', + ], + 'scripts': [ 'flutter/lib/web_ui/dev/felt' ], + }, + ] + }, }; } From 465dd03748e13033e9f6c68af0a18a78cc13e316 Mon Sep 17 00:00:00 2001 From: Yegor Jbanov Date: Thu, 12 Dec 2024 16:54:01 -0800 Subject: [PATCH 2/5] bringup: true --- .ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.ci.yaml b/.ci.yaml index 74449eb6a538a..065e17c939a1b 100644 --- a/.ci.yaml +++ b/.ci.yaml @@ -381,6 +381,7 @@ targets: clobber: "true" - name: Linux linux_web_engine_build + bringup: true recipe: engine_v2/engine_v2 timeout: 120 properties: From 285f7ada911839ab129ffdd26b73195747c97858 Mon Sep 17 00:00:00 2001 From: Yegor Jbanov Date: Fri, 13 Dec 2024 11:00:51 -0800 Subject: [PATCH 3/5] swap names --- .ci.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.ci.yaml b/.ci.yaml index 065e17c939a1b..fdac6349508c9 100644 --- a/.ci.yaml +++ b/.ci.yaml @@ -380,8 +380,7 @@ targets: config_name: linux_license clobber: "true" - - name: Linux linux_web_engine_build - bringup: true + - name: Linux linux_web_engine recipe: engine_v2/engine_v2 timeout: 120 properties: @@ -406,7 +405,8 @@ targets: - ci/** - flutter_frontend_server/** - - name: Linux linux_web_engine + - name: Linux linux_web_engine_build + bringup: true recipe: engine_v2/engine_v2 timeout: 120 properties: From 6673e10dbf60c731096abf7c1a7abff3b05c6784 Mon Sep 17 00:00:00 2001 From: Yegor Jbanov Date: Fri, 13 Dec 2024 12:52:11 -0800 Subject: [PATCH 4/5] keep the old build step name --- ci/builders/linux_web_engine_build.json | 2 +- lib/web_ui/dev/generate_builder_json.dart | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/ci/builders/linux_web_engine_build.json b/ci/builders/linux_web_engine_build.json index 9bc499584aa50..488589a49bd3b 100644 --- a/ci/builders/linux_web_engine_build.json +++ b/ci/builders/linux_web_engine_build.json @@ -3,7 +3,7 @@ "_comment2": "See `generate_builder_json.dart` for the generator code", "builds": [ { - "name": "web_build/artifacts", + "name": "web_tests/artifacts", "drone_dimensions": [ "device_type=none", "os=Linux", diff --git a/lib/web_ui/dev/generate_builder_json.dart b/lib/web_ui/dev/generate_builder_json.dart index 93e244550d401..d4230603e0dbe 100644 --- a/lib/web_ui/dev/generate_builder_json.dart +++ b/lib/web_ui/dev/generate_builder_json.dart @@ -61,7 +61,7 @@ class GenerateBuilderJsonCommand extends Command { '_comment': 'THIS IS A GENERATED FILE. Do not edit this file directly.', '_comment2': 'See `generate_builder_json.dart` for the generator code', 'builds': [ - _getArtifactBuildStep(name: 'web_build/artifacts', forTesting: false), + _getArtifactBuildStep(forTesting: false), ], }; return const JsonEncoder.withIndent(' ').convert(outputJson); @@ -72,7 +72,7 @@ class GenerateBuilderJsonCommand extends Command { '_comment': 'THIS IS A GENERATED FILE. Do not edit this file directly.', '_comment2': 'See `generate_builder_json.dart` for the generator code', 'builds': [ - _getArtifactBuildStep(name: 'web_tests/artifacts', forTesting: true), + _getArtifactBuildStep(forTesting: true), if (true) for (final TestBundle bundle in config.testBundles) _getBundleBuildStep(bundle), @@ -83,11 +83,10 @@ class GenerateBuilderJsonCommand extends Command { } Map _getArtifactBuildStep({ - required String name, required bool forTesting, }) { return { - 'name': name, + 'name': 'web_tests/artifacts', 'drone_dimensions': [ 'device_type=none', 'os=Linux', From 9fd0b5236e8864f4c22a5c72a0f7857525a3cced Mon Sep 17 00:00:00 2001 From: Yegor Jbanov Date: Fri, 13 Dec 2024 13:30:49 -0800 Subject: [PATCH 5/5] readd copy step --- ci/builders/linux_web_engine_build.json | 16 +++++++++++++- lib/web_ui/dev/generate_builder_json.dart | 27 ++++++++++++----------- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/ci/builders/linux_web_engine_build.json b/ci/builders/linux_web_engine_build.json index 488589a49bd3b..6b7f2430ee881 100644 --- a/ci/builders/linux_web_engine_build.json +++ b/ci/builders/linux_web_engine_build.json @@ -35,7 +35,21 @@ ], "realm": "production" } - ] + ], + "generators": { + "tasks": [ + { + "name": "copy artifacts for web tests", + "parameters": [ + "test", + "--copy-artifacts" + ], + "scripts": [ + "flutter/lib/web_ui/dev/felt" + ] + } + ] + } } ] } diff --git a/lib/web_ui/dev/generate_builder_json.dart b/lib/web_ui/dev/generate_builder_json.dart index d4230603e0dbe..f9d324f1fcf20 100644 --- a/lib/web_ui/dev/generate_builder_json.dart +++ b/lib/web_ui/dev/generate_builder_json.dart @@ -120,9 +120,9 @@ class GenerateBuilderJsonCommand extends Command { 'realm': 'production', } ], - if (forTesting) - 'generators': { - 'tasks': [ + 'generators': { + 'tasks': [ + if (forTesting) { 'name': 'check licenses', 'parameters': [ @@ -130,6 +130,7 @@ class GenerateBuilderJsonCommand extends Command { ], 'scripts': [ 'flutter/lib/web_ui/dev/felt' ], }, + if (forTesting) { 'name': 'web engine analysis', 'parameters': [ @@ -137,16 +138,16 @@ class GenerateBuilderJsonCommand extends Command { ], 'scripts': [ 'flutter/lib/web_ui/dev/felt' ], }, - { - 'name': 'copy artifacts for web tests', - 'parameters': [ - 'test', - '--copy-artifacts', - ], - 'scripts': [ 'flutter/lib/web_ui/dev/felt' ], - }, - ] - }, + { + 'name': 'copy artifacts for web tests', + 'parameters': [ + 'test', + '--copy-artifacts', + ], + 'scripts': [ 'flutter/lib/web_ui/dev/felt' ], + }, + ] + }, }; }