feat: add artifact_proxy server

.github/workflows/deploy_artifacts_proxy.yaml

@@ -0,0 +1,62 @@
+name: Deploy Artifacts Proxy
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - .github/workflows/deploy_artifacts_proxy.yaml
+      - packages/artifacts_proxy/**"
+
+env:
+  PROJECT_ID: code-push-prod
+  SERVICE: artifacts-proxy
+  REGION: us-central1
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+
+    defaults:
+      run:
+        working-directory: packages/artifacts_proxy
+
+    name: ☁️ Artifacts Proxy
+
+    steps:
+      - name: 📚 Git Checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - uses: dart-lang/setup-dart@v1
+
+      - name: Setup Cloud SDK
+        uses: google-github-actions/[email protected]
+        with:
+          project_id: ${{ env.PROJECT_ID }}
+          service_account_key: ${{ secrets.CLOUD_RUN_SA_PROD }}
+          export_default_credentials: true
+
+      - name: Authorize Docker Push
+        run: gcloud auth configure-docker
+
+      - name: Build and Push Container
+        run: |-
+          docker build -t gcr.io/${{ env.PROJECT_ID }}/${{ env.SERVICE }}:${{  github.sha }} .
+          docker push gcr.io/${{ env.PROJECT_ID }}/${{ env.SERVICE }}:${{  github.sha }}
+
+      - name: Deploy to Cloud Run
+        id: deploy
+        uses: google-github-actions/[email protected]
+        with:
+          service: ${{ env.SERVICE }}
+          image: gcr.io/${{ env.PROJECT_ID }}/${{ env.SERVICE }}:${{  github.sha }}
+          region: ${{ env.REGION }}
+
+      - name: Show Output
+        run: echo ${{ steps.deploy.outputs.url }}
+
+      - name: Ping
+        run: curl "${{ steps.deploy.outputs.url }}"

.github/workflows/main.yaml

@@ -32,10 +32,13 @@ jobs:
         id: needs_dart_build
         with:
           filters: |
+            artifacts_proxy:
+              - ./.github/actions/dart_package
+              - packages/artifacts_proxy/**
             shorebird_cli:
               - ./.github/actions/dart_package
               - packages/shorebird_cli/**
-              - packages/shorebird_code_push_client/**            
+              - packages/shorebird_code_push_client/**
             shorebird_code_push_client:
               - ./.github/actions/dart_package
               - packages/shorebird_code_push_client/**

packages/artifact_proxy/.gitignore

@@ -0,0 +1,9 @@
+# https://dart.dev/guides/libraries/private-files
+# Created by `dart pub`
+.dart_tool/
+
+# Used for local testing of artifact_proxy.
+artifacts/
+
+# Test related files
+coverage/

packages/artifact_proxy/Dockerfile

@@ -0,0 +1,23 @@
+# Official Dart image: https://hub.docker.com/_/dart
+# Specify the Dart SDK base image version using dart:<version> (ex: dart:2.12)
+FROM dart:stable AS build
+
+# Resolve app dependencies.
+WORKDIR /app
+
+# Copy app source code and AOT compile it.
+COPY . .
+
+# Ensure packages are still up-to-date if anything has changed
+RUN dart pub get --offline
+RUN dart compile exe bin/server.dart -o bin/server
+
+# Build minimal serving image from AOT-compiled `/server` and required system
+# libraries and configuration files stored in `/runtime/` from the build stage.
+FROM scratch
+COPY --from=build /runtime/ /
+COPY --from=build /app/bin/server /app/bin/
+
+# Start server.
+EXPOSE 8080
+CMD ["/app/bin/server"]

packages/artifact_proxy/README.md

@@ -0,0 +1,26 @@
+# Artifact Proxy
+
+This is a tool for proxying Flutter artifacts from a derived Flutter engine
+revision back to the base Flutter engine revision. This is useful for
+when you need to modify _some_ of the Flutter artifacts but not all of them.
+
+This is a development tool which map requests to Google
+Storage (either Shorebird's bucket or the official Flutter buckets).
+
+## Usage
+
+Uses `config.yaml` to configure the engine revisions and artifact overrides.
+
+```bash
+# Run locally with hot-reload enabled.
+dart --enable-vm-service run bin/server.dart --watch
+```
+
+And then in a separate terminal:
+
+```
+FLUTTER_STORAGE_BASE_URL=http://localhost:8080 flutter precache -a
+```
+
+You should use a separate checkout of Flutter when running this, so you don't
+poison the cache of your main Flutter checkout.

packages/artifact_proxy/analysis_options.yaml

@@ -0,0 +1 @@
+include: package:very_good_analysis/analysis_options.4.0.0.yaml

packages/artifact_proxy/bin/server.dart

@@ -0,0 +1,55 @@
+// ignore_for_file: avoid_print
+
+import 'dart:io';
+
+import 'package:args/args.dart';
+import 'package:artifact_proxy/artifact_proxy.dart';
+import 'package:shelf/shelf.dart';
+import 'package:shelf/shelf_io.dart' as shelf_io;
+import 'package:shelf_hotreload/shelf_hotreload.dart';
+import 'package:yaml/yaml.dart';
+
+Future<void> main(List<String> args) async {
+  final parser = ArgParser()
+    ..addOption(
+      'config',
+      defaultsTo: 'config.yaml',
+      help: 'Path to config file.',
+    )
+    ..addFlag(
+      'record',
+      help: 'Record requests into config file.',
+    )
+    ..addFlag(
+      'watch',
+      help: 'Whether to watch for changes and hot-reload.',
+    );
+
+  final results = parser.parse(args);
+
+  if (results.rest.isNotEmpty) {
+    print(parser.usage);
+    exit(1);
+  }
+
+  final shouldWatch = results['watch'] as bool;
+  final configPath = results['config'] as String;
+  final config = loadYaml(File(configPath).readAsStringSync()) as Map;
+  final handler = artifactProxyHandler(config: config);
+  final ip = InternetAddress.anyIPv6;
+  final port = int.parse(Platform.environment['PORT'] ?? '8080');
+
+  // Hot-reload is enabled when the `--watch` flag is passed.
+  if (shouldWatch) return withHotreload(() => serve(handler, ip, port));
+
+  await serve(handler, ip, port);
+}
+
+Future<HttpServer> serve(Handler proxy, InternetAddress ip, int port) async {
+  const pipeline = Pipeline();
+  final handler = pipeline.addMiddleware(logRequests()).addHandler(proxy);
+  final server = await shelf_io.serve(handler, ip, port);
+  print('Serving at http://localhost:${server.port}');
+  server.autoCompress = true;
+  return server;
+}

packages/artifact_proxy/config.yaml

@@ -0,0 +1,23 @@
+engine_mappings:
+  79f4c5321a581f580a9bda01ec372cbf4a53aa53:
+    flutter_engine_revision: 9aa7816315095c86410527932918c718cb35e7d6
+    shorebird_storage_bucket: download.shorebird.dev
+    shorebird_artifact_overrides:
+      # android-x64 artifacts.zip
+      - flutter_infra_release/flutter/$engine/android-x64-release/artifacts.zip
+
+      # embedding release
+      - download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.pom
+      - download.flutter.io/io/flutter/flutter_embedding_release/1.0.0-$engine/flutter_embedding_release-1.0.0-$engine.jar
+
+      # arm64_v8a release
+      - download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.pom
+      - download.flutter.io/io/flutter/arm64_v8a_release/1.0.0-$engine/arm64_v8a_release-1.0.0-$engine.jar
+
+      # armeabi_v7a release
+      - download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.pom
+      - download.flutter.io/io/flutter/armeabi_v7a_release/1.0.0-$engine/armeabi_v7a_release-1.0.0-$engine.jar
+
+      # x86_64 release
+      - download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.pom
+      - download.flutter.io/io/flutter/x86_64_release/1.0.0-$engine/x86_64_release-1.0.0-$engine.jar

packages/artifact_proxy/lib/artifact_proxy.dart

@@ -0,0 +1,87 @@
+// ignore_for_file: avoid_print
+
+import 'package:collection/collection.dart';
+import 'package:shelf/shelf.dart';
+
+/// A [Handler] that proxies artifact requests to the correct location.
+/// This is determined based on the [config].
+Handler artifactProxyHandler({required Map<dynamic, dynamic> config}) {
+  final engineMappings = config['engine_mappings'] as Map;
+  final shorebirdEngineRevisions = engineMappings.keys.cast<String>();
+
+  return (Request request) {
+    final path = request.url.path;
+    final shorebirdEngineRevision = shorebirdEngineRevisions.firstWhereOrNull(
+      path.contains,
+    );
+
+    final normalizedPath = shorebirdEngineRevision != null
+        ? path.replaceAll(shorebirdEngineRevision, r'$engine')
+        : path;
+
+    if (shorebirdEngineRevision == null) {
+      final location = getFlutterArtifactLocation(artifactPath: normalizedPath);
+      print('No engine revision detected, forwarding to: $location');
+      return Response.found(location);
+    }
+
+    final engineMapping = engineMappings[shorebirdEngineRevision] as Map;
+    final shorebirdOverrides =
+        engineMapping['shorebird_artifact_overrides'] as List;
+    final flutterEngineRevision =
+        engineMapping['flutter_engine_revision'] as String;
+    final shorebirdStorageBucket =
+        engineMapping['shorebird_storage_bucket'] as String;
+    final shouldOverride = shorebirdOverrides.contains(normalizedPath);
+
+    if (shouldOverride) {
+      final location = getShorebirdArtifactLocation(
+        artifactPath: normalizedPath,
+        engine: shorebirdEngineRevision,
+        bucket: shorebirdStorageBucket,
+      );
+      print('Shorebird artifact detected, forwarding to: $location');
+      return Response.found(location);
+    }
+
+    final location = getFlutterArtifactLocation(
+      artifactPath: normalizedPath,
+      engine: flutterEngineRevision,
+    );
+    print('Flutter artifact detected, forwarding to: $location');
+    return Response.found(location);
+  };
+}
+
+/// Returns the location of the artifact at [artifactPath] using the
+/// specified [engine] revision for original Flutter artifacts.
+String getFlutterArtifactLocation({
+  required String artifactPath,
+  String? engine,
+}) {
+  final adjustedPath = engine != null
+      ? artifactPath.replaceAll(r'$engine', engine)
+      : artifactPath;
+
+  final isChromeInfra = adjustedPath.contains('flutter_infra_release/cipd');
+
+  if (isChromeInfra) {
+    return adjustedPath.replaceAll(
+      'flutter_infra_release/cipd',
+      'https://chrome-infra-packages.appspot.com/dl',
+    );
+  }
+
+  return 'https://storage.googleapis.com/$adjustedPath';
+}
+
+/// Returns the location of the artifact at [artifactPath] using the
+/// specified [engine] revision for Shorebird artifacts.
+String getShorebirdArtifactLocation({
+  required String artifactPath,
+  required String engine,
+  required String bucket,
+}) {
+  final adjustedPath = artifactPath.replaceAll(r'$engine', engine);
+  return 'https://storage.googleapis.com/$bucket/$adjustedPath';
+}