WIP converting image build logic to pure GH actions

Signed-off-by: Matt Oswalt <[email protected]>

.github/workflows/generate-preview.yml

@@ -5,14 +5,23 @@ on:
     branches: [master]
     types: 
       - completed
+inputs:
+
+  # TODO - note in https://github.blog/2020-08-03-github-actions-improvements-for-fork-and-pull-request-workflows/ this can be invoked
+  # via a webhook as well. This may be necessary to communicate the pr number as an input - you MUST have this.
+  #
+  # TODO - for security reasons (that we will absolutely want) this will run on the default branch. However we still want to check out the PR branch
+  # so we can build the images. So we need the PR number
+  pr_number:
+    description: 'Number of the PR to check out'
+    required: true
 
 jobs:
   prebuild:
     runs-on: ubuntu-latest
     steps:
 
-    # TODO - for security reasons (that we will absolutely want) this will run on the default branch. However we still want to check out the PR branch
-    # so we can build the images.
+
 
     - uses: actions/checkout@v2
 

.github/workflows/main.yml

@@ -2,9 +2,13 @@ name: CI
 on:
   pull_request:
     branches: [ master ]
+env: 
+  emptyArray: "[]"
+  emptyLiteral: "empty"
+  retagSource: "v1.3.0"
 
 jobs:
-  build:
+  prebuild:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
@@ -28,6 +32,160 @@ jobs:
       run: ./check-changelog.sh
       if: ${{ success() }}
 
+
+    # https://github.blog/2020-08-03-github-actions-improvements-for-fork-and-pull-request-workflows/
+
+    # WRITE SCRIPTS AS ACTIONS
+    # https://github.com/actions/checkout/issues/331
+    # https://github.com/actions/github-script
+
+    # EXPRESSIONS CHEAT SHEET
+    # https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions
+
+
+    - name: Create Preview
+      id: create_preview
+      run: |
+        echo ::set-output name=preview_id::$(./create-preview.sh | jq -r '.ID')
+
+    - name: Get PR number
+      id: get_pr_number
+      run: |
+        echo ::set-output name=pr_number::$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH")
+      if: ${{ success() }}
+
+    - name: Check out PR
+      run: |
+        hub pr checkout ${{ steps.get_pr_number.outputs.pr_number }}
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+    # TODO - DON"T FORGET TO USE A SVC ACCOUNT THAT CAN ONLY PULL FROM THE IMAGES BUCKET, READ ONLY
+    # https://github.com/marketplace/actions/google-cloud-platform-gcp-cli-gcloud
+    # - name: Authenticate on GCS
+    #   uses: actions-hub/gcloud@master
+    #   env:
+    #     PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
+    #     APPLICATION_CREDENTIALS: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }}
+    #   with:
+    #     args: info
+
+    - name: Get Buildables
+      id: get_buildables
+      run: |
+        echo ::set-output name=buildables::$(cd images && find . -maxdepth 2 -type f -name 'Makefile' -printf '%h;' | tr -d './' | rev | cut -c 2- | rev | jq -Rc 'split(";")')
+
+    - name: Get Changed
+      id: get_changed
+      run: |
+        echo ::set-output name=changed::$(git diff --name-only master..HEAD images/ | sed -rn 's/images\/([^/]*)\/.*/\\1/p' | tr '\n' ';' | rev | cut -c 2- | rev)
+
+    # https://github.com/actions/github-script
+    # TODO - may also be useful for updating status? The GH javascript library is installed
+    - name: Get Image Directives
+      uses: actions/github-script@v3
+      id: get_image_directives
+      with:
+        script: |
+          var to_retag = [
+            "vqfx-snap1",
+            "vqfx-snap2",
+            "vqfx-snap3",
+          ]
+          var to_build = []
+
+          var buildables = ${{ steps.get_buildables.outputs.buildables  || env.emptyArray }};
+          var changed = ${{ steps.get_changed.outputs.changed || env.emptyArray }};
+
+          buildables.forEach(function(buildable) {
+            if (!changed.includes(buildable)) {
+              console.log(buildable + " not found in changed, adding to to_retag");
+              to_retag.push(buildable);
+            } else {
+              console.log(buildable + " was found in changed, adding to to_build");
+              to_build.push(buildable);
+            }
+          });
+
+          // The strategy matrix can't handle empty arrays so we'll add this here and check for it during the retag/build jobs
+          if (to_build.length == 0) {
+            to_build.push("empty")
+          }
+          if (to_retag.length == 0) {
+            to_retag.push("empty")
+          }
+
+          return {
+            "to_build": to_build,
+            "to_retag": to_retag
+          }
+
+    # - name: Debug output
+    #   run: |
+    #     echo ${{ steps.get_image_directives.outputs.result }}
+    # - name: Debug retags
+    #   run: |
+    #     echo ${{ fromJSON(steps.get_image_directives.outputs.result).to_retag }}
+    # - name: Debug builds
+    #   run: |
+    #     echo ${{ fromJSON(steps.get_image_directives.outputs.result).to_build }}
+
+
+    # https://github.com/google-github-actions/setup-gcloud
+    - name: Set up Cloud SDK
+      uses: google-github-actions/setup-gcloud@master
+      with:
+        project_id: ${{ secrets.GCP_PROJECT_ID }}
+        service_account_key: ${{ secrets.GCP_SA_KEY }}
+        export_default_credentials: true
+
+    # - name: Debug gsutil
+    #   run: |
+    #     gsutil cp "gs://nrelabs-curriculum-base-images/crpd/junos-routing-crpd-docker-20.2R1.10.tgz" "./crpd.tgz"
+
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v1
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+
+    - name: Login to DockerHub
+      uses: docker/login-action@v1
+      with:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+    # https://stackoverflow.com/questions/59175332/using-output-from-a-previous-job-in-a-new-one-in-a-github-action/61236803#61236803
+    outputs:
+      preview_id: ${{ steps.create_preview.outputs.preview_id }}
+      pr_number: ${{ steps.get_pr_number.outputs.pr_number }}
+      image_directives: ${{ steps.get_image_directives.outputs.result }}
+
+  build_image:
+    needs: prebuild
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        image_to_build: ${{ fromJson(needs.prebuild.outputs.image_directives).to_build }}
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build image
+      run: |
+        cd images/${{ matrix.image_to_build }} && make dockerfast TARGET_VERSION=${{ env.retagSource }}
+      if: ${{ matrix.image_to_build != env.emptyLiteral }}       # Ugly hack to prevent this from running when the matrix array is "empty"
+
+  retag_image:
+    needs: prebuild
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        image_to_retag: ${{ fromJson(needs.prebuild.outputs.image_directives).to_retag }}
+    steps:
+    - uses: actions/checkout@v2
+    - name: Re-tag image
+      run: |
+        docker pull antidotelabs/${{ matrix.image_to_retag }}:${{ env.retagSource }} && docker tag antidotelabs/${{ matrix.image_to_retag }}:${{ env.retagSource }} antidotelabs/${{ matrix.image_to_retag }}:${{ needs.prebuild.outputs.preview_id }} && docker push antidotelabs/${{ matrix.image_to_retag }}:${{ needs.prebuild.outputs.preview_id }}
+      if: ${{ matrix.image_to_build != env.emptyLiteral }}        # Ugly hack to prevent this from running when the matrix array is "empty"
+
   # ---------------------------
 
 

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## In development
 
 - Adding image build to preview pipeline [#352](https://github.com/nre-learning/nrelabs-curriculum/pull/352)
+- More image build changes (moving to GH actions) [#354](https://github.com/nre-learning/nrelabs-curriculum/pull/354)
 
 ## v1.3.0 - December 13, 2020