From b2f8cbef92f487a0399c051b3b3c42132a876016 Mon Sep 17 00:00:00 2001 From: Ross Wang Date: Fri, 11 Aug 2023 18:20:05 -0700 Subject: [PATCH] [ci] Only cache on postsubmit Writes to a cache from a pull request are not visible elsewhere, so * Build the caches in postsubmit so that they will be available eventually * While we're at it, run all tests as a postsubmit in case of rebase issues. * Don't save caches in the PR so that they won't spam persistence. * Also don't bother generating an emulator image for caching on PR workflows. https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#restrictions-for-accessing-a-cache While we're at it, switch to environments for better security since secrets could otherwise be available to pull requests before review, and add the patrol cache. Fun fact: https://github.com/actions/runner/issues/2238 --- .github/actions/avd/action.yml | 10 ++- .github/actions/flutter/action.yml | 25 +++++- .../workflows/{pull_request.yml => ci.yml} | 77 ++++++++++++++++++- .github/workflows/deploy_alpha.yml | 66 ---------------- 4 files changed, 104 insertions(+), 74 deletions(-) rename .github/workflows/{pull_request.yml => ci.yml} (73%) delete mode 100644 .github/workflows/deploy_alpha.yml diff --git a/.github/actions/avd/action.yml b/.github/actions/avd/action.yml index d55148d..7226545 100644 --- a/.github/actions/avd/action.yml +++ b/.github/actions/avd/action.yml @@ -13,6 +13,10 @@ inputs: target: description: Target image required: true + cache-save: + description: Whether or not to save the image. + required: false + default: ${{ github.event_name == 'push' }} runs: using: composite steps: @@ -25,7 +29,7 @@ runs: ~/.android/adb* key: avd-${{ inputs.profile }}-${{ inputs.api-level }}-${{ inputs.arch }}-${{ inputs.target }} - name: Create AVD and generate snapshot for caching - if: steps.avd-cache.outputs.cache-hit != 'true' + if: ${{ !steps.avd-cache.outputs.cache-hit && input.cache-save == 'true' }} uses: reactivecircus/android-emulator-runner@v2 with: profile: ${{ inputs.profile }} @@ -36,10 +40,10 @@ runs: emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim script: echo "Generated AVD snapshot for caching." - name: AVD cache save - if: steps.avd-cache.outputs.cache-hit != 'true' + if: ${{ !steps.avd-cache.outputs.cache-hit && inputs.cache-save == 'true' }} uses: actions/cache/save@v3 with: path: | ~/.android/avd/* ~/.android/adb* - key: ${{ steps.avd-cache.outputs.cache-primary-key }} \ No newline at end of file + key: ${{ steps.avd-cache.outputs.cache-primary-key }} diff --git a/.github/actions/flutter/action.yml b/.github/actions/flutter/action.yml index ed96fd6..5a37f11 100644 --- a/.github/actions/flutter/action.yml +++ b/.github/actions/flutter/action.yml @@ -1,5 +1,10 @@ name: Flutter description: Flutter checkout and PATH setup +inputs: + cache-save: + description: Whether or not to save the flutter repo and pub caches. + required: false + default: ${{ github.event_name == 'push' }} runs: using: composite steps: @@ -18,7 +23,7 @@ runs: path: flutter key: flutter@${{ steps.ls-remote.outputs.hash }} - name: Checkout - if: steps.checkout-cache.outputs.cache-hit != 'true' + if: ${{ !steps.checkout-cache.outputs.cache-hit }} uses: actions/checkout@v3 with: repository: AsturaPhoenix/flutter @@ -26,15 +31,27 @@ runs: path: flutter fetch-depth: 0 # to fetch tags to determine the Flutter version - name: Flutter checkout save - if: steps.checkout-cache.outputs.cache-hit != 'true' + if: ${{ !steps.checkout-cache.outputs.cache-hit && inputs.cache-save }} uses: actions/cache/save@v3 with: path: flutter key: ${{ steps.checkout-cache.outputs.cache-primary-key }} - name: Flutter/pub cache + if: inputs.cache-save == 'true' uses: actions/cache@v3 - id: pub-cache + with: + path: | + flutter/bin/cache + ~/.pub-cache + key: pub-${{ runner.os }}-${{ runner.arch }}-${{ steps.ls-remote.outputs.hash }}-${{ github.job }}-${{ github.sha }} + restore-keys: | + pub-${{ runner.os }}-${{ runner.arch }}-${{ steps.ls-remote.outputs.hash }}-${{ github.job }}- + pub-${{ runner.os }}-${{ runner.arch }}-${{ steps.ls-remote.outputs.hash }}- + pub-${{ runner.os }}-${{ runner.arch }}- + - name: Flutter/pub cache restore + if: inputs.cache-save != 'true' + uses: actions/cache/restore@v3 with: path: | flutter/bin/cache @@ -47,4 +64,4 @@ runs: - name: PATH run: '{ echo "$GITHUB_WORKSPACE/flutter/bin"; echo "$HOME/.pub-cache/bin"; } >> $GITHUB_PATH' - shell: bash \ No newline at end of file + shell: bash diff --git a/.github/workflows/pull_request.yml b/.github/workflows/ci.yml similarity index 73% rename from .github/workflows/pull_request.yml rename to .github/workflows/ci.yml index 0faaef1..ca7149a 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,8 @@ -name: Pull request +name: CI on: + push: + branches: [master] pull_request: branches: [master] @@ -180,6 +182,11 @@ jobs: - run: flutter pub get + - name: Patrol cache restore + uses: actions/cache/restore@v3 + with: + path: ~/.cache/patrol + key: patrol-cache - name: Download native test apk uses: actions/download-artifact@v3 with: @@ -196,3 +203,71 @@ jobs: emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim working-directory: trip_planner_aquamarine/app script: dart run patrol_cli:main drive -t integration_test/native_test.dart --use-application-binary build/app/outputs/flutter-apk/app-debug.apk + - name: Patrol cache save + if: github.event_name == 'push' + uses: actions/cache/save@v3 + with: + path: ~/.cache/patrol + key: patrol-cache + + deploy-alpha: + name: Deploy alpha + runs-on: ubuntu-latest + environment: alpha + if: github.event_name == 'push' + needs: linux-tests + concurrency: deploy-alpha + defaults: + run: + working-directory: trip_planner_aquamarine + + steps: + - name: Check out repo + uses: actions/checkout@v3 + with: + path: trip_planner_aquamarine + - name: Set up Flutter + uses: ./trip_planner_aquamarine/.github/actions/flutter + + - run: cd app && flutter build web --base-href "/trip_planner_aquamarine/" --web-renderer canvaskit + - name: Package web build + run: cd app/build && tar -czf web.tar.gz web + + - run: docker build -t aquamarine_server . + - run: docker save aquamarine_server -o aquamarine_server.tar + + - name: Set up SSH + run: | + mkdir ~/.ssh + cd ~/.ssh + echo '${{ secrets.KNOWN_HOSTS }}' >> known_hosts + echo '${{ secrets.SSH }}' >> id_rsa + chmod 600 id_rsa + + - name: Transfer web build + run: 'scp app/build/web.tar.gz imagipioneer@${{ vars.HOSTNAME }}:' + - name: Unpack and flip + run: | + ssh imagipioneer@${{ vars.HOSTNAME }} ' + tar -xvf web.tar.gz && + rm -rf trip_planner_aquamarine && + mv web trip_planner_aquamarine && + rm web.tar.gz' + + - name: Transfer server + run: 'scp aquamarine_server.tar ${{ vars.USERNAME }}@${{ vars.HOSTNAME }}:aquamarine_server' + - name: Unpack and flip + run: | + ssh imagipioneer@${{ vars.HOSTNAME }} ' + # Prune now to retain the previous image for rollbacks + sudo docker image prune -f; + cd aquamarine_server && + sudo docker load -i aquamarine_server.tar && + rm aquamarine_server.tar && + sudo docker rm -f aquamarine_server; + sudo docker run \ + --name aquamarine_server \ + -d \ + -p 1080:1080 \ + --mount type=bind,src=./persistence,dst=/server/persistence \ + aquamarine_server' diff --git a/.github/workflows/deploy_alpha.yml b/.github/workflows/deploy_alpha.yml deleted file mode 100644 index 85efd2c..0000000 --- a/.github/workflows/deploy_alpha.yml +++ /dev/null @@ -1,66 +0,0 @@ -name: Deploy alpha - -concurrency: deploy-alpha - -on: - push: - branches: [master] - -jobs: - deploy-alpha: - name: Deploy alpha - runs-on: ubuntu-latest - defaults: - run: - working-directory: trip_planner_aquamarine - - steps: - - name: Check out repo - uses: actions/checkout@v3 - with: - path: trip_planner_aquamarine - - name: Set up Flutter - uses: ./trip_planner_aquamarine/.github/actions/flutter - - - run: cd app && flutter build web --base-href "/trip_planner_aquamarine/" --web-renderer canvaskit - - name: Package web build - run: cd app/build && tar --create -f web.tar.gz web - - - run: docker build -t aquamarine_server . - - run: docker save aquamarine_server -o aquamarine_server.tar - - - name: Set up SSH - run: | - mkdir ~/.ssh - cd ~/.ssh - echo '${{ secrets.ALPHA_KNOWN_HOSTS }}' >> known_hosts - echo '${{ secrets.ALPHA_SSH }}' >> id_rsa - chmod 600 id_rsa - - - name: Transfer web build - run: 'scp app/build/web.tar.gz imagipioneer@${{ vars.ALPHA_IP }}:' - - name: Unpack and flip - run: | - ssh imagipioneer@${{ vars.ALPHA_IP }} ' - tar -xvf web.tar.gz && - rm -rf trip_planner_aquamarine && - mv web trip_planner_aquamarine && - rm web.tar.gz' - - - name: Transfer server - run: 'scp aquamarine_server.tar imagipioneer@${{ vars.ALPHA_IP }}:aquamarine_server' - - name: Unpack and flip - run: | - ssh imagipioneer@${{ vars.ALPHA_IP }} ' - # Prune now to retain the previous image for rollbacks - sudo docker image prune -f; - cd aquamarine_server && - sudo docker load -i aquamarine_server.tar && - rm aquamarine_server.tar && - sudo docker rm -f aquamarine_server; - sudo docker run \ - --name aquamarine_server \ - -d \ - -p 1080:1080 \ - --mount type=bind,src=./persistence,dst=/server/persistence \ - aquamarine_server'