From d8ce9b1a72614d435e84b72946e3aa43d715097d Mon Sep 17 00:00:00 2001 From: Matt Oswalt Date: Sat, 20 Mar 2021 15:03:15 -0400 Subject: [PATCH] WIP converting image build logic to pure GH actions Signed-off-by: Matt Oswalt --- .github/workflows/generate-preview.yml | 13 +- .github/workflows/main.yml | 160 ++++++++++++++++++++++++- CHANGELOG.md | 1 + 3 files changed, 171 insertions(+), 3 deletions(-) diff --git a/.github/workflows/generate-preview.yml b/.github/workflows/generate-preview.yml index d63111c1..f897dc53 100644 --- a/.github/workflows/generate-preview.yml +++ b/.github/workflows/generate-preview.yml @@ -5,14 +5,23 @@ on: branches: [master] types: - completed +inputs: + + # TODO - note in https://github.blog/2020-08-03-github-actions-improvements-for-fork-and-pull-request-workflows/ this can be invoked + # via a webhook as well. This may be necessary to communicate the pr number as an input - you MUST have this. + # + # TODO - for security reasons (that we will absolutely want) this will run on the default branch. However we still want to check out the PR branch + # so we can build the images. So we need the PR number + pr_number: + description: 'Number of the PR to check out' + required: true jobs: prebuild: runs-on: ubuntu-latest steps: - # TODO - for security reasons (that we will absolutely want) this will run on the default branch. However we still want to check out the PR branch - # so we can build the images. + - uses: actions/checkout@v2 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c72ee19a..fe8bf451 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -2,9 +2,13 @@ name: CI on: pull_request: branches: [ master ] +env: + emptyArray: "[]" + emptyLiteral: "empty" + retagSource: "v1.3.0" jobs: - build: + prebuild: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -28,6 +32,160 @@ jobs: run: ./check-changelog.sh if: ${{ success() }} + + # https://github.blog/2020-08-03-github-actions-improvements-for-fork-and-pull-request-workflows/ + + # WRITE SCRIPTS AS ACTIONS + # https://github.com/actions/checkout/issues/331 + # https://github.com/actions/github-script + + # EXPRESSIONS CHEAT SHEET + # https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions + + + - name: Create Preview + id: create_preview + run: | + echo ::set-output name=preview_id::$(./create-preview.sh | jq -r '.ID') + + - name: Get PR number + id: get_pr_number + run: | + echo ::set-output name=pr_number::$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH") + if: ${{ success() }} + + - name: Check out PR + run: | + hub pr checkout ${{ steps.get_pr_number.outputs.pr_number }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # TODO - DON"T FORGET TO USE A SVC ACCOUNT THAT CAN ONLY PULL FROM THE IMAGES BUCKET, READ ONLY + # https://github.com/marketplace/actions/google-cloud-platform-gcp-cli-gcloud + # - name: Authenticate on GCS + # uses: actions-hub/gcloud@master + # env: + # PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} + # APPLICATION_CREDENTIALS: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }} + # with: + # args: info + + - name: Get Buildables + id: get_buildables + run: | + echo ::set-output name=buildables::$(cd images && find . -maxdepth 2 -type f -name 'Makefile' -printf '%h;' | tr -d './' | rev | cut -c 2- | rev | jq -Rc 'split(";")') + + - name: Get Changed + id: get_changed + run: | + echo ::set-output name=changed::$(git diff --name-only master..HEAD images/ | sed -rn 's/images\/([^/]*)\/.*/\\1/p' | tr '\n' ';' | rev | cut -c 2- | rev) + + # https://github.com/actions/github-script + # TODO - may also be useful for updating status? The GH javascript library is installed + - name: Get Image Directives + uses: actions/github-script@v3 + id: get_image_directives + with: + script: | + var to_retag = [ + "vqfx-snap1", + "vqfx-snap2", + "vqfx-snap3", + ] + var to_build = [] + + var buildables = ${{ steps.get_buildables.outputs.buildables || env.emptyArray }}; + var changed = ${{ steps.get_changed.outputs.changed || env.emptyArray }}; + + buildables.forEach(function(buildable) { + if (!changed.includes(buildable)) { + console.log(buildable + " not found in changed, adding to to_retag"); + to_retag.push(buildable); + } else { + console.log(buildable + " was found in changed, adding to to_build"); + to_build.push(buildable); + } + }); + + // The strategy matrix can't handle empty arrays so we'll add this here and check for it during the retag/build jobs + if (to_build.length == 0) { + to_build.push("empty") + } + if (to_retag.length == 0) { + to_retag.push("empty") + } + + return { + "to_build": to_build, + "to_retag": to_retag + } + + # - name: Debug output + # run: | + # echo ${{ steps.get_image_directives.outputs.result }} + # - name: Debug retags + # run: | + # echo ${{ fromJSON(steps.get_image_directives.outputs.result).to_retag }} + # - name: Debug builds + # run: | + # echo ${{ fromJSON(steps.get_image_directives.outputs.result).to_build }} + + + # https://github.com/google-github-actions/setup-gcloud + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@master + with: + project_id: ${{ secrets.GCP_PROJECT_ID }} + service_account_key: ${{ secrets.GCP_SA_KEY }} + export_default_credentials: true + + # - name: Debug gsutil + # run: | + # gsutil cp "gs://nrelabs-curriculum-base-images/crpd/junos-routing-crpd-docker-20.2R1.10.tgz" "./crpd.tgz" + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + # https://stackoverflow.com/questions/59175332/using-output-from-a-previous-job-in-a-new-one-in-a-github-action/61236803#61236803 + outputs: + preview_id: ${{ steps.create_preview.outputs.preview_id }} + pr_number: ${{ steps.get_pr_number.outputs.pr_number }} + image_directives: ${{ steps.get_image_directives.outputs.result }} + + build_image: + needs: prebuild + runs-on: ubuntu-latest + strategy: + matrix: + image_to_build: ${{ fromJson(needs.prebuild.outputs.image_directives).to_build }} + steps: + - uses: actions/checkout@v2 + - name: Build image + run: | + cd images/${{ matrix.image_to_build }} && make dockerfast TARGET_VERSION=${{ env.retagSource }} + if: ${{ matrix.image_to_build != env.emptyLiteral }} # Ugly hack to prevent this from running when the matrix array is "empty" + + retag_image: + needs: prebuild + runs-on: ubuntu-latest + strategy: + matrix: + image_to_retag: ${{ fromJson(needs.prebuild.outputs.image_directives).to_retag }} + steps: + - uses: actions/checkout@v2 + - name: Re-tag image + run: | + docker pull antidotelabs/${{ matrix.image_to_retag }}:${{ env.retagSource }} && docker tag antidotelabs/${{ matrix.image_to_retag }}:${{ env.retagSource }} antidotelabs/${{ matrix.image_to_retag }}:${{ needs.prebuild.outputs.preview_id }} && docker push antidotelabs/${{ matrix.image_to_retag }}:${{ needs.prebuild.outputs.preview_id }} + if: ${{ matrix.image_to_build != env.emptyLiteral }} # Ugly hack to prevent this from running when the matrix array is "empty" + # --------------------------- diff --git a/CHANGELOG.md b/CHANGELOG.md index b192c77b..d929fc70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## In development - Adding image build to preview pipeline [#352](https://github.com/nre-learning/nrelabs-curriculum/pull/352) +- More image build changes (moving to GH actions) [#354](https://github.com/nre-learning/nrelabs-curriculum/pull/354) ## v1.3.0 - December 13, 2020