Skip to content

Commit

Permalink
Fetch from GCS buckets.
Browse files Browse the repository at this point in the history
  • Loading branch information
eyebrowsoffire committed Dec 17, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent 652a754 commit e4c4260
Showing 5 changed files with 151 additions and 83 deletions.
15 changes: 15 additions & 0 deletions lib/web_ui/dev/build.dart
Original file line number Diff line number Diff line change
@@ -68,6 +68,21 @@ class BuildCommand extends Command<bool> with ArgUtils<bool> {
List<String> get targets => argResults?.rest ?? <String>[];
bool get embedDwarf => boolArg('dwarf');

RuntimeMode get runtimeMode {
final bool isProfile = boolArg('profile');
final bool isDebug = boolArg('debug');
if (isProfile && isDebug) {
throw ToolExit('Cannot specify both --profile and --debug at the same time.');
}
if (isProfile) {
return RuntimeMode.profile;
} else if (isDebug) {
return RuntimeMode.debug;
} else {
return RuntimeMode.release;
}
}

@override
FutureOr<bool> run() async {
if (embedDwarf && runtimeMode != RuntimeMode.debug) {
15 changes: 14 additions & 1 deletion lib/web_ui/dev/common.dart
Original file line number Diff line number Diff line change
@@ -11,6 +11,7 @@ import 'browser.dart';
import 'chrome.dart';
import 'edge.dart';
import 'environment.dart';
import 'exceptions.dart';
import 'felt_config.dart';
import 'firefox.dart';
import 'safari_macos.dart';
@@ -225,7 +226,6 @@ class LuciConfig {
LuciConfig(this.realm);

factory LuciConfig.fromJson(String contextJson) {
print('luci context json: $contextJson');
final json = jsonDecode(contextJson) as Map<String, Object?>;
final LuciRealm realm = switch ((json['realm'] as Map<String, Object?>?)?['name']) {
'flutter:prod' => LuciRealm.Prod,
@@ -254,6 +254,19 @@ bool get isLuci => io.Platform.environment['LUCI_CONTEXT'] != null;
/// environements.
bool get isCi => isLuci;

final String gitRevision = () {
final result = io.Process.runSync(
'git',
<String>['rev-parse', 'HEAD'],
workingDirectory: path.join(environment.engineSrcDir.path, 'flutter'),
stderrEncoding: utf8,
stdoutEncoding: utf8);
if (result.exitCode != 0) {
throw ToolExit('Failed to get git revision. Exit code: ${result.exitCode} Error: ${result.stderr}');
}
return (result.stdout as String).trim();
}();

const String kChrome = 'chrome';
const String kEdge = 'edge';
const String kFirefox = 'firefox';
135 changes: 70 additions & 65 deletions lib/web_ui/dev/steps/copy_artifacts_step.dart
Original file line number Diff line number Diff line change
@@ -5,6 +5,8 @@
import 'dart:convert' show JsonEncoder;
import 'dart:io' as io;

import 'package:archive/archive_io.dart';
import 'package:http/http.dart' as http;
import 'package:path/path.dart' as pathlib;

import '../common.dart';
@@ -14,11 +16,25 @@ import '../felt_config.dart';
import '../pipeline.dart';
import '../utils.dart';

sealed class ArtifactSource {}

class LocalArtifactSource implements ArtifactSource {
LocalArtifactSource({required this.mode});

final RuntimeMode mode;
}

class GcsArtifactSource implements ArtifactSource {
GcsArtifactSource({required this.realm});

final LuciRealm realm;
}

class CopyArtifactsStep implements PipelineStep {
CopyArtifactsStep(this.artifactDeps, { required this.runtimeMode });
CopyArtifactsStep(this.artifactDeps, { required this.source });

final ArtifactDependencies artifactDeps;
final RuntimeMode runtimeMode;
final ArtifactSource source;

@override
String get description => 'copy_artifacts';
@@ -31,26 +47,65 @@ class CopyArtifactsStep implements PipelineStep {
await cleanup();
}

Future<io.Directory> _downloadArtifacts(LuciRealm realm) async {
final String realmComponent = switch (realm) {
LuciRealm.Prod => '',
LuciRealm.Staging || LuciRealm.Try => 'flutter_archives_v2/',
LuciRealm.Unknown => throw ToolExit('Could not generate artifact bucket url for unknown realm.'),
};
final Uri url = Uri.https('storage.googleapis.com', '${realmComponent}flutter_infra_release/flutter/$gitRevision/flutter-web-sdk.zip');
final http.Response response = await http.Client().get(url);
if (response.statusCode != 200) {
throw ToolExit('Could not download flutter-web-sdk.zip from cloud bucket at URL: $url. Response status code: ${response.statusCode}');
}
final Archive archive = ZipDecoder().decodeBytes(response.bodyBytes);
final io.Directory tempDirectory = await io.Directory.systemTemp.createTemp();
await extractArchiveToDisk(archive, tempDirectory.absolute.path);
return tempDirectory;
}

@override
Future<void> run() async {
print('LUCI realm: ${luciConfig?.realm}');
final String flutterJsSourceDirectory;
final String canvaskitSourceDirectory;
final String canvaskitChromiumSourceDirectory;
final String skwasmSourceDirectory;
final String skwasmStSourceDirectory;
switch (source) {
case LocalArtifactSource(:final mode):
final buildDirectory = getBuildDirectoryForRuntimeMode(mode).path;
flutterJsSourceDirectory = pathlib.join(buildDirectory, 'flutter_web_sdk', 'flutter_js');
canvaskitSourceDirectory = pathlib.join(buildDirectory, 'canvaskit');
canvaskitChromiumSourceDirectory = pathlib.join(buildDirectory, 'canvaskit_chromium');
skwasmSourceDirectory = pathlib.join(buildDirectory, 'skwasm');
skwasmStSourceDirectory = pathlib.join(buildDirectory, 'skwasm_st');

case GcsArtifactSource(:final realm):
final artifactsDirectory = (await _downloadArtifacts(realm)).path;
flutterJsSourceDirectory = pathlib.join(artifactsDirectory, 'flutter_js');
canvaskitSourceDirectory = pathlib.join(artifactsDirectory, 'canvaskit');
canvaskitChromiumSourceDirectory = pathlib.join(artifactsDirectory, 'canvaskit', 'chromium');
skwasmSourceDirectory = pathlib.join(artifactsDirectory, 'canvaskit');
skwasmStSourceDirectory = pathlib.join(artifactsDirectory, 'canvaskit');
}

await environment.webTestsArtifactsDir.create(recursive: true);
await buildHostPage();
await copyTestFonts();
await copySkiaTestImages();
await copyFlutterJsFiles();
await copyFlutterJsFiles(flutterJsSourceDirectory);
if (artifactDeps.canvasKit) {
print('Copying CanvasKit...');
await copyCanvasKitFiles('canvaskit', 'canvaskit');
await copyWasmLibrary('canvaskit', canvaskitSourceDirectory, 'canvaskit');
}
if (artifactDeps.canvasKitChromium) {
print('Copying CanvasKit (Chromium)...');
await copyCanvasKitFiles('canvaskit_chromium', 'canvaskit/chromium');
await copyWasmLibrary('canvaskit', canvaskitChromiumSourceDirectory, 'canvaskit/chromium');
}
if (artifactDeps.skwasm) {
print('Copying Skwasm...');
await copySkwasm();
await copyWasmLibrary('skwasm', skwasmSourceDirectory, 'canvaskit');
await copyWasmLibrary('skwasm_st', skwasmStSourceDirectory, 'canvaskit');
}
}

@@ -154,12 +209,8 @@ class CopyArtifactsStep implements PipelineStep {
}
}

Future<void> copyFlutterJsFiles() async {
final io.Directory flutterJsInputDirectory = io.Directory(pathlib.join(
outBuildPath,
'flutter_web_sdk',
'flutter_js',
));
Future<void> copyFlutterJsFiles(String sourcePath) async {
final io.Directory flutterJsInputDirectory = io.Directory(sourcePath);
final String targetDirectoryPath = pathlib.join(
environment.webTestsArtifactsDir.path,
'flutter_js',
@@ -185,24 +236,19 @@ class CopyArtifactsStep implements PipelineStep {
}
}

Future<void> copyCanvasKitFiles(String sourcePath, String destinationPath) async {
final String sourceDirectoryPath = pathlib.join(
outBuildPath,
sourcePath,
);

Future<void> copyWasmLibrary(String libraryName, String sourcePath, String destinationPath) async {
final String targetDirectoryPath = pathlib.join(
environment.webTestsArtifactsDir.path,
destinationPath,
);

for (final String filename in <String>[
'canvaskit.js',
'canvaskit.wasm',
'canvaskit.wasm.map',
'$libraryName.js',
'$libraryName.wasm',
'$libraryName.wasm.map',
]) {
final io.File sourceFile = io.File(pathlib.join(
sourceDirectoryPath,
sourcePath,
filename,
));
final io.File targetFile = io.File(pathlib.join(
@@ -215,55 +261,14 @@ class CopyArtifactsStep implements PipelineStep {
// they are optional.
continue;
} {
throw ToolExit('Built CanvasKit artifact not found at path "$sourceFile".');
throw ToolExit('Built artifact not found at path "$sourceFile".');
}
}
await targetFile.create(recursive: true);
await sourceFile.copy(targetFile.path);
}
}

String get outBuildPath => getBuildDirectoryForRuntimeMode(runtimeMode).path;

Future<void> copySkwasm() async {
final io.Directory targetDir = io.Directory(pathlib.join(
environment.webTestsArtifactsDir.path,
'canvaskit',
));

await targetDir.create(recursive: true);

for (final String fileName in <String>[
'skwasm.wasm',
'skwasm.wasm.map',
'skwasm.js',
'skwasm_st.wasm',
'skwasm_st.wasm.map',
'skwasm_st.js',
]) {
final io.File sourceFile = io.File(pathlib.join(
outBuildPath,
'flutter_web_sdk',
'canvaskit',
fileName,
));
if (!sourceFile.existsSync()) {
if (fileName.endsWith('.map')) {
// Sourcemaps are only generated under certain build conditions, so
// they are optional.
continue;
} {
throw ToolExit('Built Skwasm artifact not found at path "$sourceFile".');
}
}
final io.File targetFile = io.File(pathlib.join(
targetDir.path,
fileName,
));
await sourceFile.copy(targetFile.path);
}
}

Future<void> buildHostPage() async {
final String hostDartPath = pathlib.join('lib', 'static', 'host.dart');
final io.File hostDartFile = io.File(pathlib.join(
53 changes: 52 additions & 1 deletion lib/web_ui/dev/test_runner.dart
Original file line number Diff line number Diff line change
@@ -10,6 +10,7 @@ import 'package:path/path.dart' as path;

import 'package:watcher/src/watch_event.dart';

import 'common.dart';
import 'environment.dart';
import 'exceptions.dart';
import 'felt_config.dart';
@@ -76,6 +77,22 @@ class TestCommand extends Command<bool> with ArgUtils<bool> {
'debug',
help: 'Use artifacts from the debug build instead of release.'
)
..addFlag(
'release',
help: 'Use artifacts from the release build. This is the default.'
)
..addFlag(
'gcs-prod',
help: 'Use artifacts from the prod gcs bucket populated by CI.'
)
..addFlag(
'gcs-staging',
help: 'Use artifacts from the prod gcs bucket populated by CI.'
)
..addFlag(
'gcs-try',
help: 'Use artifacts from the prod gcs bucket populated by CI.'
)
..addFlag(
'dwarf',
help: 'Debug wasm modules using embedded DWARF data.'
@@ -318,6 +335,40 @@ class TestCommand extends Command<bool> with ArgUtils<bool> {
(ArtifactDependencies deps, TestSuite suite) => deps | suite.artifactDependencies);
}

ArtifactSource get artifactSource {
final List<ArtifactSource> sources = <ArtifactSource>[];
if (boolArg('debug')) {
sources.add(LocalArtifactSource(mode: RuntimeMode.debug));
}
if (boolArg('profile')) {
sources.add(LocalArtifactSource(mode: RuntimeMode.profile));
}
if (boolArg('release')) {
sources.add(LocalArtifactSource(mode: RuntimeMode.release));
}
if (boolArg('gcs-prod')) {
sources.add(GcsArtifactSource(realm: LuciRealm.Prod));
}
if (boolArg('gcs-staging')) {
sources.add(GcsArtifactSource(realm: LuciRealm.Staging));
}
if (boolArg('gcs-try')) {
sources.add(GcsArtifactSource(realm: LuciRealm.Try));
}
if (sources.length > 1) {
throw ToolExit('Cannot specify more than one artifact source.');
}
if (sources.length == 1) {
return sources.first;
}
final realm = luciConfig?.realm;
if (realm != null) {
return GcsArtifactSource(realm: realm);
} else {
return LocalArtifactSource(mode: RuntimeMode.release);
}
}

@override
Future<bool> run() async {
final List<TestSuite> filteredSuites = _filterTestSuites();
@@ -360,7 +411,7 @@ class TestCommand extends Command<bool> with ArgUtils<bool> {
final Set<FilePath>? testFiles = targetFiles.isEmpty ? null : Set<FilePath>.from(targetFiles);
final Pipeline testPipeline = Pipeline(steps: <PipelineStep>[
if (isWatchMode) ClearTerminalScreenStep(),
if (shouldCopyArtifacts) CopyArtifactsStep(artifacts, runtimeMode: runtimeMode),
if (shouldCopyArtifacts) CopyArtifactsStep(artifacts, source: artifactSource),
if (shouldCompile)
for (final TestBundle bundle in bundles)
CompileBundleStep(
16 changes: 0 additions & 16 deletions lib/web_ui/dev/utils.dart
Original file line number Diff line number Diff line change
@@ -12,7 +12,6 @@ import 'package:path/path.dart' as path;

import 'common.dart';
import 'environment.dart';
import 'exceptions.dart';
import 'felt_config.dart';

enum RuntimeMode {
@@ -312,21 +311,6 @@ mixin ArgUtils<T> on Command<T> {

/// Extracts a string argument from [argResults].
String stringArg(String name) => argResults![name] as String;

RuntimeMode get runtimeMode {
final bool isProfile = boolArg('profile');
final bool isDebug = boolArg('debug');
if (isProfile && isDebug) {
throw ToolExit('Cannot specify both --profile and --debug at the same time.');
}
if (isProfile) {
return RuntimeMode.profile;
} else if (isDebug) {
return RuntimeMode.debug;
} else {
return RuntimeMode.release;
}
}
}

io.Directory getBuildDirectoryForRuntimeMode(RuntimeMode runtimeMode) =>

0 comments on commit e4c4260

Please sign in to comment.