From dc825c5e59156a2db3b3440f5bf8e8bf0e444159 Mon Sep 17 00:00:00 2001 From: Jackson Gardner Date: Mon, 16 Dec 2024 16:28:39 -0800 Subject: [PATCH 1/4] Detect luci realm. --- lib/web_ui/dev/common.dart | 33 +++++++++++++++++++ lib/web_ui/dev/steps/copy_artifacts_step.dart | 3 ++ 2 files changed, 36 insertions(+) diff --git a/lib/web_ui/dev/common.dart b/lib/web_ui/dev/common.dart index ab6c9ea32ee85..417f9c3342661 100644 --- a/lib/web_ui/dev/common.dart +++ b/lib/web_ui/dev/common.dart @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +import 'dart:convert'; import 'dart:io' as io; import 'package:path/path.dart' as path; @@ -213,6 +214,38 @@ class DevNull implements StringSink { void writeln([Object? obj = '']) {} } +enum LuciRealm { + Prod, + Staging, + Try, + Unknown, +} + +class LuciConfig { + LuciConfig(this.realm); + + factory LuciConfig.fromJson(String contextJson) { + final json = jsonDecode(contextJson) as Map; + final LuciRealm realm = switch (json['realm']) { + 'flutter:prod' => LuciRealm.Prod, + 'flutter:staging' => LuciRealm.Staging, + 'flutter:try' => LuciRealm.Try, + _ => LuciRealm.Unknown, + }; + return LuciConfig(realm); + } + + final LuciRealm realm; +} + +final LuciConfig? luciConfig = () { + final String? contextPath = io.Platform.environment['LUCI_CONTEXT']; + if (contextPath == null) { + return null; + } + return LuciConfig.fromJson(io.File(contextPath).readAsStringSync()); +}(); + /// Whether the felt command is running on LUCI. bool get isLuci => io.Platform.environment['LUCI_CONTEXT'] != null; diff --git a/lib/web_ui/dev/steps/copy_artifacts_step.dart b/lib/web_ui/dev/steps/copy_artifacts_step.dart index 930b30ac9f894..e3cee3eddb395 100644 --- a/lib/web_ui/dev/steps/copy_artifacts_step.dart +++ b/lib/web_ui/dev/steps/copy_artifacts_step.dart @@ -7,6 +7,7 @@ import 'dart:io' as io; import 'package:path/path.dart' as pathlib; +import '../common.dart'; import '../environment.dart'; import '../exceptions.dart'; import '../felt_config.dart'; @@ -32,6 +33,8 @@ class CopyArtifactsStep implements PipelineStep { @override Future run() async { + print('LUCI realm: ${luciConfig?.realm}'); + await environment.webTestsArtifactsDir.create(recursive: true); await buildHostPage(); await copyTestFonts(); From 1a4ca3cdb82dbff5070bc3f4451987b2f6760a7f Mon Sep 17 00:00:00 2001 From: Jackson Gardner Date: Mon, 16 Dec 2024 16:54:07 -0800 Subject: [PATCH 2/4] Check context string. --- lib/web_ui/dev/common.dart | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/web_ui/dev/common.dart b/lib/web_ui/dev/common.dart index 417f9c3342661..c78641fa922a8 100644 --- a/lib/web_ui/dev/common.dart +++ b/lib/web_ui/dev/common.dart @@ -225,6 +225,7 @@ class LuciConfig { LuciConfig(this.realm); factory LuciConfig.fromJson(String contextJson) { + print('luci context json: $contextJson'); final json = jsonDecode(contextJson) as Map; final LuciRealm realm = switch (json['realm']) { 'flutter:prod' => LuciRealm.Prod, From 652a754414e9335298b2dc99cba9957d41fbfa25 Mon Sep 17 00:00:00 2001 From: Jackson Gardner Date: Mon, 16 Dec 2024 17:13:00 -0800 Subject: [PATCH 3/4] Correctly parse realm. --- lib/web_ui/dev/common.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/web_ui/dev/common.dart b/lib/web_ui/dev/common.dart index c78641fa922a8..1509a198bb31a 100644 --- a/lib/web_ui/dev/common.dart +++ b/lib/web_ui/dev/common.dart @@ -227,7 +227,7 @@ class LuciConfig { factory LuciConfig.fromJson(String contextJson) { print('luci context json: $contextJson'); final json = jsonDecode(contextJson) as Map; - final LuciRealm realm = switch (json['realm']) { + final LuciRealm realm = switch ((json['realm'] as Map?)?['name']) { 'flutter:prod' => LuciRealm.Prod, 'flutter:staging' => LuciRealm.Staging, 'flutter:try' => LuciRealm.Try, From e4c42604dbddf7eb44b44bcf34dc4191615759a5 Mon Sep 17 00:00:00 2001 From: Jackson Gardner Date: Tue, 17 Dec 2024 14:03:44 -0800 Subject: [PATCH 4/4] Fetch from GCS buckets. --- lib/web_ui/dev/build.dart | 15 ++ lib/web_ui/dev/common.dart | 15 +- lib/web_ui/dev/steps/copy_artifacts_step.dart | 135 +++++++++--------- lib/web_ui/dev/test_runner.dart | 53 ++++++- lib/web_ui/dev/utils.dart | 16 --- 5 files changed, 151 insertions(+), 83 deletions(-) diff --git a/lib/web_ui/dev/build.dart b/lib/web_ui/dev/build.dart index 58ca394dc9bb3..f1c1278085f46 100644 --- a/lib/web_ui/dev/build.dart +++ b/lib/web_ui/dev/build.dart @@ -68,6 +68,21 @@ class BuildCommand extends Command with ArgUtils { List get targets => argResults?.rest ?? []; bool get embedDwarf => boolArg('dwarf'); + RuntimeMode get runtimeMode { + final bool isProfile = boolArg('profile'); + final bool isDebug = boolArg('debug'); + if (isProfile && isDebug) { + throw ToolExit('Cannot specify both --profile and --debug at the same time.'); + } + if (isProfile) { + return RuntimeMode.profile; + } else if (isDebug) { + return RuntimeMode.debug; + } else { + return RuntimeMode.release; + } + } + @override FutureOr run() async { if (embedDwarf && runtimeMode != RuntimeMode.debug) { diff --git a/lib/web_ui/dev/common.dart b/lib/web_ui/dev/common.dart index 1509a198bb31a..535876041d831 100644 --- a/lib/web_ui/dev/common.dart +++ b/lib/web_ui/dev/common.dart @@ -11,6 +11,7 @@ import 'browser.dart'; import 'chrome.dart'; import 'edge.dart'; import 'environment.dart'; +import 'exceptions.dart'; import 'felt_config.dart'; import 'firefox.dart'; import 'safari_macos.dart'; @@ -225,7 +226,6 @@ class LuciConfig { LuciConfig(this.realm); factory LuciConfig.fromJson(String contextJson) { - print('luci context json: $contextJson'); final json = jsonDecode(contextJson) as Map; final LuciRealm realm = switch ((json['realm'] as Map?)?['name']) { 'flutter:prod' => LuciRealm.Prod, @@ -254,6 +254,19 @@ bool get isLuci => io.Platform.environment['LUCI_CONTEXT'] != null; /// environements. bool get isCi => isLuci; +final String gitRevision = () { + final result = io.Process.runSync( + 'git', + ['rev-parse', 'HEAD'], + workingDirectory: path.join(environment.engineSrcDir.path, 'flutter'), + stderrEncoding: utf8, + stdoutEncoding: utf8); + if (result.exitCode != 0) { + throw ToolExit('Failed to get git revision. Exit code: ${result.exitCode} Error: ${result.stderr}'); + } + return (result.stdout as String).trim(); +}(); + const String kChrome = 'chrome'; const String kEdge = 'edge'; const String kFirefox = 'firefox'; diff --git a/lib/web_ui/dev/steps/copy_artifacts_step.dart b/lib/web_ui/dev/steps/copy_artifacts_step.dart index e3cee3eddb395..89c6878e81ff2 100644 --- a/lib/web_ui/dev/steps/copy_artifacts_step.dart +++ b/lib/web_ui/dev/steps/copy_artifacts_step.dart @@ -5,6 +5,8 @@ import 'dart:convert' show JsonEncoder; import 'dart:io' as io; +import 'package:archive/archive_io.dart'; +import 'package:http/http.dart' as http; import 'package:path/path.dart' as pathlib; import '../common.dart'; @@ -14,11 +16,25 @@ import '../felt_config.dart'; import '../pipeline.dart'; import '../utils.dart'; +sealed class ArtifactSource {} + +class LocalArtifactSource implements ArtifactSource { + LocalArtifactSource({required this.mode}); + + final RuntimeMode mode; +} + +class GcsArtifactSource implements ArtifactSource { + GcsArtifactSource({required this.realm}); + + final LuciRealm realm; +} + class CopyArtifactsStep implements PipelineStep { - CopyArtifactsStep(this.artifactDeps, { required this.runtimeMode }); + CopyArtifactsStep(this.artifactDeps, { required this.source }); final ArtifactDependencies artifactDeps; - final RuntimeMode runtimeMode; + final ArtifactSource source; @override String get description => 'copy_artifacts'; @@ -31,26 +47,65 @@ class CopyArtifactsStep implements PipelineStep { await cleanup(); } + Future _downloadArtifacts(LuciRealm realm) async { + final String realmComponent = switch (realm) { + LuciRealm.Prod => '', + LuciRealm.Staging || LuciRealm.Try => 'flutter_archives_v2/', + LuciRealm.Unknown => throw ToolExit('Could not generate artifact bucket url for unknown realm.'), + }; + final Uri url = Uri.https('storage.googleapis.com', '${realmComponent}flutter_infra_release/flutter/$gitRevision/flutter-web-sdk.zip'); + final http.Response response = await http.Client().get(url); + if (response.statusCode != 200) { + throw ToolExit('Could not download flutter-web-sdk.zip from cloud bucket at URL: $url. Response status code: ${response.statusCode}'); + } + final Archive archive = ZipDecoder().decodeBytes(response.bodyBytes); + final io.Directory tempDirectory = await io.Directory.systemTemp.createTemp(); + await extractArchiveToDisk(archive, tempDirectory.absolute.path); + return tempDirectory; + } + @override Future run() async { - print('LUCI realm: ${luciConfig?.realm}'); + final String flutterJsSourceDirectory; + final String canvaskitSourceDirectory; + final String canvaskitChromiumSourceDirectory; + final String skwasmSourceDirectory; + final String skwasmStSourceDirectory; + switch (source) { + case LocalArtifactSource(:final mode): + final buildDirectory = getBuildDirectoryForRuntimeMode(mode).path; + flutterJsSourceDirectory = pathlib.join(buildDirectory, 'flutter_web_sdk', 'flutter_js'); + canvaskitSourceDirectory = pathlib.join(buildDirectory, 'canvaskit'); + canvaskitChromiumSourceDirectory = pathlib.join(buildDirectory, 'canvaskit_chromium'); + skwasmSourceDirectory = pathlib.join(buildDirectory, 'skwasm'); + skwasmStSourceDirectory = pathlib.join(buildDirectory, 'skwasm_st'); + + case GcsArtifactSource(:final realm): + final artifactsDirectory = (await _downloadArtifacts(realm)).path; + flutterJsSourceDirectory = pathlib.join(artifactsDirectory, 'flutter_js'); + canvaskitSourceDirectory = pathlib.join(artifactsDirectory, 'canvaskit'); + canvaskitChromiumSourceDirectory = pathlib.join(artifactsDirectory, 'canvaskit', 'chromium'); + skwasmSourceDirectory = pathlib.join(artifactsDirectory, 'canvaskit'); + skwasmStSourceDirectory = pathlib.join(artifactsDirectory, 'canvaskit'); + } await environment.webTestsArtifactsDir.create(recursive: true); await buildHostPage(); await copyTestFonts(); await copySkiaTestImages(); - await copyFlutterJsFiles(); + await copyFlutterJsFiles(flutterJsSourceDirectory); if (artifactDeps.canvasKit) { print('Copying CanvasKit...'); - await copyCanvasKitFiles('canvaskit', 'canvaskit'); + await copyWasmLibrary('canvaskit', canvaskitSourceDirectory, 'canvaskit'); } if (artifactDeps.canvasKitChromium) { print('Copying CanvasKit (Chromium)...'); - await copyCanvasKitFiles('canvaskit_chromium', 'canvaskit/chromium'); + await copyWasmLibrary('canvaskit', canvaskitChromiumSourceDirectory, 'canvaskit/chromium'); } if (artifactDeps.skwasm) { print('Copying Skwasm...'); - await copySkwasm(); + await copyWasmLibrary('skwasm', skwasmSourceDirectory, 'canvaskit'); + await copyWasmLibrary('skwasm_st', skwasmStSourceDirectory, 'canvaskit'); } } @@ -154,12 +209,8 @@ class CopyArtifactsStep implements PipelineStep { } } - Future copyFlutterJsFiles() async { - final io.Directory flutterJsInputDirectory = io.Directory(pathlib.join( - outBuildPath, - 'flutter_web_sdk', - 'flutter_js', - )); + Future copyFlutterJsFiles(String sourcePath) async { + final io.Directory flutterJsInputDirectory = io.Directory(sourcePath); final String targetDirectoryPath = pathlib.join( environment.webTestsArtifactsDir.path, 'flutter_js', @@ -185,24 +236,19 @@ class CopyArtifactsStep implements PipelineStep { } } - Future copyCanvasKitFiles(String sourcePath, String destinationPath) async { - final String sourceDirectoryPath = pathlib.join( - outBuildPath, - sourcePath, - ); - + Future copyWasmLibrary(String libraryName, String sourcePath, String destinationPath) async { final String targetDirectoryPath = pathlib.join( environment.webTestsArtifactsDir.path, destinationPath, ); for (final String filename in [ - 'canvaskit.js', - 'canvaskit.wasm', - 'canvaskit.wasm.map', + '$libraryName.js', + '$libraryName.wasm', + '$libraryName.wasm.map', ]) { final io.File sourceFile = io.File(pathlib.join( - sourceDirectoryPath, + sourcePath, filename, )); final io.File targetFile = io.File(pathlib.join( @@ -215,7 +261,7 @@ class CopyArtifactsStep implements PipelineStep { // they are optional. continue; } { - throw ToolExit('Built CanvasKit artifact not found at path "$sourceFile".'); + throw ToolExit('Built artifact not found at path "$sourceFile".'); } } await targetFile.create(recursive: true); @@ -223,47 +269,6 @@ class CopyArtifactsStep implements PipelineStep { } } - String get outBuildPath => getBuildDirectoryForRuntimeMode(runtimeMode).path; - - Future copySkwasm() async { - final io.Directory targetDir = io.Directory(pathlib.join( - environment.webTestsArtifactsDir.path, - 'canvaskit', - )); - - await targetDir.create(recursive: true); - - for (final String fileName in [ - 'skwasm.wasm', - 'skwasm.wasm.map', - 'skwasm.js', - 'skwasm_st.wasm', - 'skwasm_st.wasm.map', - 'skwasm_st.js', - ]) { - final io.File sourceFile = io.File(pathlib.join( - outBuildPath, - 'flutter_web_sdk', - 'canvaskit', - fileName, - )); - if (!sourceFile.existsSync()) { - if (fileName.endsWith('.map')) { - // Sourcemaps are only generated under certain build conditions, so - // they are optional. - continue; - } { - throw ToolExit('Built Skwasm artifact not found at path "$sourceFile".'); - } - } - final io.File targetFile = io.File(pathlib.join( - targetDir.path, - fileName, - )); - await sourceFile.copy(targetFile.path); - } - } - Future buildHostPage() async { final String hostDartPath = pathlib.join('lib', 'static', 'host.dart'); final io.File hostDartFile = io.File(pathlib.join( diff --git a/lib/web_ui/dev/test_runner.dart b/lib/web_ui/dev/test_runner.dart index 672c70f837f72..1c78e388f2f39 100644 --- a/lib/web_ui/dev/test_runner.dart +++ b/lib/web_ui/dev/test_runner.dart @@ -10,6 +10,7 @@ import 'package:path/path.dart' as path; import 'package:watcher/src/watch_event.dart'; +import 'common.dart'; import 'environment.dart'; import 'exceptions.dart'; import 'felt_config.dart'; @@ -76,6 +77,22 @@ class TestCommand extends Command with ArgUtils { 'debug', help: 'Use artifacts from the debug build instead of release.' ) + ..addFlag( + 'release', + help: 'Use artifacts from the release build. This is the default.' + ) + ..addFlag( + 'gcs-prod', + help: 'Use artifacts from the prod gcs bucket populated by CI.' + ) + ..addFlag( + 'gcs-staging', + help: 'Use artifacts from the prod gcs bucket populated by CI.' + ) + ..addFlag( + 'gcs-try', + help: 'Use artifacts from the prod gcs bucket populated by CI.' + ) ..addFlag( 'dwarf', help: 'Debug wasm modules using embedded DWARF data.' @@ -318,6 +335,40 @@ class TestCommand extends Command with ArgUtils { (ArtifactDependencies deps, TestSuite suite) => deps | suite.artifactDependencies); } + ArtifactSource get artifactSource { + final List sources = []; + if (boolArg('debug')) { + sources.add(LocalArtifactSource(mode: RuntimeMode.debug)); + } + if (boolArg('profile')) { + sources.add(LocalArtifactSource(mode: RuntimeMode.profile)); + } + if (boolArg('release')) { + sources.add(LocalArtifactSource(mode: RuntimeMode.release)); + } + if (boolArg('gcs-prod')) { + sources.add(GcsArtifactSource(realm: LuciRealm.Prod)); + } + if (boolArg('gcs-staging')) { + sources.add(GcsArtifactSource(realm: LuciRealm.Staging)); + } + if (boolArg('gcs-try')) { + sources.add(GcsArtifactSource(realm: LuciRealm.Try)); + } + if (sources.length > 1) { + throw ToolExit('Cannot specify more than one artifact source.'); + } + if (sources.length == 1) { + return sources.first; + } + final realm = luciConfig?.realm; + if (realm != null) { + return GcsArtifactSource(realm: realm); + } else { + return LocalArtifactSource(mode: RuntimeMode.release); + } + } + @override Future run() async { final List filteredSuites = _filterTestSuites(); @@ -360,7 +411,7 @@ class TestCommand extends Command with ArgUtils { final Set? testFiles = targetFiles.isEmpty ? null : Set.from(targetFiles); final Pipeline testPipeline = Pipeline(steps: [ if (isWatchMode) ClearTerminalScreenStep(), - if (shouldCopyArtifacts) CopyArtifactsStep(artifacts, runtimeMode: runtimeMode), + if (shouldCopyArtifacts) CopyArtifactsStep(artifacts, source: artifactSource), if (shouldCompile) for (final TestBundle bundle in bundles) CompileBundleStep( diff --git a/lib/web_ui/dev/utils.dart b/lib/web_ui/dev/utils.dart index 78a79a96dbc91..897be3a88c887 100644 --- a/lib/web_ui/dev/utils.dart +++ b/lib/web_ui/dev/utils.dart @@ -12,7 +12,6 @@ import 'package:path/path.dart' as path; import 'common.dart'; import 'environment.dart'; -import 'exceptions.dart'; import 'felt_config.dart'; enum RuntimeMode { @@ -312,21 +311,6 @@ mixin ArgUtils on Command { /// Extracts a string argument from [argResults]. String stringArg(String name) => argResults![name] as String; - - RuntimeMode get runtimeMode { - final bool isProfile = boolArg('profile'); - final bool isDebug = boolArg('debug'); - if (isProfile && isDebug) { - throw ToolExit('Cannot specify both --profile and --debug at the same time.'); - } - if (isProfile) { - return RuntimeMode.profile; - } else if (isDebug) { - return RuntimeMode.debug; - } else { - return RuntimeMode.release; - } - } } io.Directory getBuildDirectoryForRuntimeMode(RuntimeMode runtimeMode) =>