Adding image build support to the preview system

Signed-off-by: Matt Oswalt <[email protected]>

.github/workflows/main.yml

@@ -1,37 +1,114 @@
 name: CI
 on:
-  push:
-    branches: [ master ]
+  # push:
+  #   branches: [ master ]
   pull_request:
     branches: [ master ]
 
 jobs:
+  prebuild:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+
+    # -----------------------------------------
+
+    # TODO - don't forget to re-enable these
+
+    # - name: Install xmllint
+    #   run: sudo apt-get install aspell aspell-en dictionaries-common
+
+    # - name: Run spellchecker
+    #   run: ./check-spelling.sh
+    #   if: ${{ success() }}
+
+    # - name: Install antidote binaries
+    #   run: curl -Lo antidote.tar.gz https://github.com/nre-learning/antidote-core/releases/download/v0.6.2/antidote-linux-amd64.tar.gz && tar xvzf antidote.tar.gz
+    #   if: ${{ success() }}
+
+    # - name: Run antidote validation
+    #   run: ./antidote validate .
+    #   if: ${{ success() }}
+
+    # - name: Check changelog
+    #   run: ./check-changelog.sh
+    #   if: ${{ success() }}
+
+    # - name: Request preview
+    #   run: ./request-preview.sh
+    #   if: ${{ success() }}
+
+    # ----------------------------------------
+
+    - name: Create Preview
+      id: create_preview
+      run: |
+        echo ::set-output name=preview_id::$(./create-preview.sh | jq -r '.ID')
+
+    - name: Get Buildables
+      id: get_buildables
+      run: |
+        echo ::set-output name=buildables::$(cd images && find . -maxdepth 2 -type f -name 'Makefile' -printf '%h;' | tr -d './' | rev | cut -c 2- | rev | jq -Rc 'split(";")')
+
+    # - name: Set up QEMU
+    #   uses: docker/setup-qemu-action@v1
+    # - name: Set up Docker Buildx
+    #   uses: docker/setup-buildx-action@v1
+
+    # - name: Login to DockerHub
+    #   uses: docker/login-action@v1
+    #   with:
+    #     username: ${{ secrets.DOCKERHUB_USERNAME }}
+    #     password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+    # https://stackoverflow.com/questions/59175332/using-output-from-a-previous-job-in-a-new-one-in-a-github-action/61236803#61236803
+    outputs:
+      preview_id: ${{ steps.create_preview.outputs.preview_id }}
+      buildables: ${{ steps.get_buildables.outputs.buildables }}
+
   build:
+    needs: prebuild
     runs-on: ubuntu-latest
 
+    strategy:
+      matrix:
+        images_to_build: ${{ fromJson(needs.prebuild.outputs.buildables) }}
+
     steps:
     - uses: actions/checkout@v2
+    - name: Build and push
+      # VERY IMPORTANT that we statically have "preview-" in the tag name so we don't conflict with existing prod images
+      # run: "cd images/${{ matrix.images_to_build }} && TARGET_VERSION=preview-$preview_id make docker"
+
+      # TODO - remove this and comment back in the "make docker equivalent" above once you're confident that the directory iteration is done properly
+      # with parallelism
+      run: "cd images/${{ matrix.images_to_build }} && TARGET_VERSION=preview-$preview_id echo $(pwd)"
+
+      env:
+        preview_id: ${{needs.prebuild.outputs.preview_id}}
 
-    - name: Install xmllint
-      run: sudo apt-get install aspell aspell-en dictionaries-common
+  retag:
+    # Some images can't be built automatically, and therefore should just be retagged from a known-good "source" tag, like a curriculum release.
 
-    - name: Run spellchecker
-      run: ./check-spelling.sh
-      if: ${{ success() }}
+    needs: prebuild
+    runs-on: ubuntu-latest
 
-    - name: Install antidote binaries
-      run: curl -Lo antidote.tar.gz https://github.com/nre-learning/antidote-core/releases/download/v0.6.2/antidote-linux-amd64.tar.gz && tar xvzf antidote.tar.gz
-      if: ${{ success() }}
+    strategy:
+      matrix:
+        images_to_retag: [
+          "vqfx-snap1",
+          "vqfx-snap2",
+          "vqfx-snap3",
+        ]
 
-    - name: Run antidote validation
-      run: ./antidote validate .
-      if: ${{ success() }}
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build and push
+      run: "cd images/${{ matrix.images_to_retag }} && docker tag antidotelabs/${{ matrix.images_to_retag }}:$retag_source antidotelabs/${{ matrix.images_to_retag }}:preview-$preview_id && docker push antidotelabs/${{ matrix.images_to_retag }}:preview-$preview_id"
 
-    - name: Check changelog
-      run: ./check-changelog.sh
-      if: ${{ success() }}
+      env:
 
-    - name: Request preview
-      run: ./request-preview.sh
-      if: ${{ success() }}
-
+        # TODO - this is a weak point, as this may need to get updated every release, if these images change. They often don't, so maybe
+        # this is okay for now, and the right thing to do is just fix these images so they can be built properly, at which point
+        # this whole job can be deleted.
+        retag_source: v1.3.0

create-preview.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+set -e +o pipefail
+
+url="https://preview.nrelabs.io/createpreview"
+
+# echo "ENV TO FOLLOW"
+# echo $(env)
+
+PR_ID=$(echo $GITHUB_REF | sed "s/refs\/pull\/\(.*\)\/merge/\1/")
+
+if [ "$GITHUB_EVENT_NAME" != "pull_request" ];
+then
+  echo "Not a PR build, skipping preview"
+  exit 0
+fi
+
+# echo "Requesting preview...."
+
+echo $(curl -s $url --header "Content-Type: application/json" \
+  --data "{
+    \"branch\":\"$GITHUB_HEAD_REF\",
+    \"pullRequest\":\"$PR_ID\",
+    \"repoSlug\":\"$GITHUB_REPOSITORY\",
+    \"prSha\":\"$GITHUB_SHA\"
+  }")
+
+exit 0

images/utility/requirements.txt

@@ -1,6 +1,6 @@
-# https://github.com/paramiko/paramiko/issues/1369
-# (I think this is fixed now, so commenting this out. I ran into issues with this restriction on)
-# cryptography==2.4.2
+# https://github.com/Azure/azure-cli/issues/16858
+cryptography==3.3.2
+
 napalm
 netmiko
 jsnapy