build #1560
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: build | |
| on: | |
| push: | |
| branches: | |
| - "**" | |
| tags: | |
| - "v*.*.*" | |
| pull_request: | |
| schedule: | |
| - cron: "0 10 * * *" # everyday at 10am | |
| repository_dispatch: | |
| # Respond to rebuild requests. See: https://github.com/cisagov/action-apb/ | |
| types: [apb] | |
| workflow_dispatch: | |
| inputs: | |
| remote-shell: | |
| description: "Debug with remote shell" | |
| required: true | |
| default: "false" | |
| image-tag: | |
| description: "Tag to apply to pushed images" | |
| required: true | |
| default: "dispatch" | |
| env: | |
| BUILDX_CACHE_DIR: ~/.cache/buildx | |
| CURL_CACHE_DIR: ~/.cache/curl | |
| IMAGE_NAME: cisagov/gatherer | |
| PIP_CACHE_DIR: ~/.cache/pip | |
| PLATFORMS: "linux/amd64,linux/arm/v6,linux/arm/v7,\ | |
| linux/arm64,linux/ppc64le,linux/s390x" | |
| PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit | |
| RUN_TMATE: ${{ secrets.RUN_TMATE }} | |
| jobs: | |
| diagnostics: | |
| name: Run diagnostics | |
| runs-on: ubuntu-latest | |
| steps: | |
| # Note that a duplicate of this step must be added at the top of | |
| # each job. | |
| - id: harden-runner | |
| name: Harden the runner | |
| uses: step-security/harden-runner@v2 | |
| with: | |
| egress-policy: audit | |
| - id: github-status | |
| name: Check GitHub status | |
| uses: crazy-max/ghaction-github-status@v3 | |
| - id: dump-context | |
| name: Dump context | |
| uses: crazy-max/ghaction-dump-context@v2 | |
| lint: | |
| # Checks out the source and runs pre-commit hooks. Detects coding errors | |
| # and style deviations. | |
| name: Lint sources | |
| needs: | |
| - diagnostics | |
| runs-on: ubuntu-latest | |
| steps: | |
| - id: harden-runner | |
| name: Harden the runner | |
| uses: step-security/harden-runner@v2 | |
| with: | |
| egress-policy: audit | |
| - id: setup-env | |
| uses: cisagov/setup-env-github-action@develop | |
| - uses: actions/checkout@v4 | |
| - id: setup-python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.11" | |
| # We need the Go version and Go cache location for the actions/cache step, | |
| # so the Go installation must happen before that. | |
| - id: setup-go | |
| uses: actions/setup-go@v4 | |
| with: | |
| # There is no expectation for actual Go code so we disable caching as | |
| # it relies on the existence of a go.sum file. | |
| cache: false | |
| go-version: "1.20" | |
| - name: Lookup Go cache directory | |
| id: go-cache | |
| run: | | |
| echo "dir=$(go env GOCACHE)" >> $GITHUB_OUTPUT | |
| - uses: actions/cache@v3 | |
| env: | |
| BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ | |
| py${{ steps.setup-python.outputs.python-version }}-\ | |
| go${{ steps.setup-go.outputs.go-version }}-\ | |
| packer${{ steps.setup-env.outputs.packer-version }}-\ | |
| tf${{ steps.setup-env.outputs.terraform-version }}-" | |
| with: | |
| # Note that the .terraform directory IS NOT included in the | |
| # cache because if we were caching, then we would need to use | |
| # the `-upgrade=true` option. This option blindly pulls down the | |
| # latest modules and providers instead of checking to see if an | |
| # update is required. That behavior defeats the benefits of caching. | |
| # so there is no point in doing it for the .terraform directory. | |
| path: | | |
| ${{ env.PIP_CACHE_DIR }} | |
| ${{ env.PRE_COMMIT_CACHE_DIR }} | |
| ${{ env.CURL_CACHE_DIR }} | |
| ${{ steps.go-cache.outputs.dir }} | |
| key: "${{ env.BASE_CACHE_KEY }}\ | |
| ${{ hashFiles('**/requirements-test.txt') }}-\ | |
| ${{ hashFiles('**/requirements.txt') }}-\ | |
| ${{ hashFiles('**/.pre-commit-config.yaml') }}" | |
| restore-keys: | | |
| ${{ env.BASE_CACHE_KEY }} | |
| - name: Setup curl cache | |
| run: mkdir -p ${{ env.CURL_CACHE_DIR }} | |
| - name: Install Packer | |
| env: | |
| PACKER_VERSION: ${{ steps.setup-env.outputs.packer-version }} | |
| run: | | |
| PACKER_ZIP="packer_${PACKER_VERSION}_linux_amd64.zip" | |
| curl --output ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \ | |
| --time-cond ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \ | |
| --location \ | |
| "https://releases.hashicorp.com/packer/${PACKER_VERSION}/${PACKER_ZIP}" | |
| sudo unzip -d /opt/packer \ | |
| ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" | |
| sudo mv /usr/local/bin/packer /usr/local/bin/packer-default | |
| sudo ln -s /opt/packer/packer /usr/local/bin/packer | |
| - uses: hashicorp/setup-terraform@v2 | |
| with: | |
| terraform_version: ${{ steps.setup-env.outputs.terraform-version }} | |
| - name: Install go-critic | |
| env: | |
| PACKAGE_URL: github.com/go-critic/go-critic/cmd/gocritic | |
| PACKAGE_VERSION: ${{ steps.setup-env.outputs.go-critic-version }} | |
| run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} | |
| - name: Install gosec | |
| env: | |
| PACKAGE_URL: github.com/securego/gosec/v2/cmd/gosec | |
| PACKAGE_VERSION: ${{ steps.setup-env.outputs.gosec-version }} | |
| run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} | |
| - name: Install shfmt | |
| env: | |
| PACKAGE_URL: mvdan.cc/sh/v3/cmd/shfmt | |
| PACKAGE_VERSION: ${{ steps.setup-env.outputs.shfmt-version }} | |
| run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} | |
| - name: Install staticcheck | |
| env: | |
| PACKAGE_URL: honnef.co/go/tools/cmd/staticcheck | |
| PACKAGE_VERSION: ${{ steps.setup-env.outputs.staticcheck-version }} | |
| run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} | |
| - name: Install Terraform-docs | |
| env: | |
| PACKAGE_URL: github.com/terraform-docs/terraform-docs | |
| PACKAGE_VERSION: ${{ steps.setup-env.outputs.terraform-docs-version }} | |
| run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip setuptools wheel | |
| pip install --upgrade --requirement requirements-test.txt | |
| - name: Set up pre-commit hook environments | |
| run: pre-commit install-hooks | |
| - name: Run pre-commit on all files | |
| run: pre-commit run --all-files | |
| - name: Setup tmate debug session | |
| uses: mxschmitt/action-tmate@v3 | |
| if: env.RUN_TMATE | |
| prepare: | |
| # Calculates and publishes outputs that are used by other jobs. | |
| # | |
| # Outputs: | |
| # created: | |
| # The current date-time in RFC3339 format. | |
| # repometa: | |
| # The json metadata describing this repository. | |
| # source_version: | |
| # The source version as reported by the `bump_version.sh show` command. | |
| # tags: | |
| # A comma separated list of Docker tags to be applied to the images on | |
| # Docker Hub. The tags will vary depending on: | |
| # - The event that triggered the build. | |
| # - The branch the build is based upon. | |
| # - The git tag the build is based upon. | |
| # | |
| # When a build is based on a git tag of the form `v*.*.*` the image will | |
| # be tagged on Docker Hub with multiple levels of version specificity. | |
| # For example, a git tag of `v1.2.3+a` will generate Docker tags of | |
| # `:1.2.3_a`, `:1.2.3`, `:1.2`, `:1`, and `:latest`. | |
| # | |
| # Builds targeting the default branch will be tagged with `:edge`. | |
| # | |
| # Builds from other branches will be tagged with the branch name. Solidi | |
| # (`/` characters - commonly known as slashes) in branch names are | |
| # replaced with hyphen-minuses (`-` characters) in the Docker tag. For | |
| # more information about the solidus see these links: | |
| # * https://www.compart.com/en/unicode/U+002F | |
| # * https://en.wikipedia.org/wiki/Slash_(punctuation)#Encoding | |
| # | |
| # Builds triggered by a push event are tagged with a short hash in the | |
| # form: sha-12345678 | |
| # | |
| # Builds triggered by a pull request are tagged with the pull request | |
| # number in the form pr-123. | |
| # | |
| # Builds triggered using the GitHub GUI (workflow_dispatch) are tagged | |
| # with the value specified by the user. | |
| # | |
| # Scheduled builds are tagged with `:nightly`. | |
| name: Prepare build variables | |
| needs: | |
| - diagnostics | |
| outputs: | |
| created: ${{ steps.prep.outputs.created }} | |
| repometa: ${{ steps.repo.outputs.result }} | |
| source_version: ${{ steps.prep.outputs.source_version }} | |
| tags: ${{ steps.prep.outputs.tags }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - id: harden-runner | |
| name: Harden the runner | |
| uses: step-security/harden-runner@v2 | |
| with: | |
| egress-policy: audit | |
| - uses: actions/checkout@v4 | |
| - name: Gather repository metadata | |
| id: repo | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const repo = await github.rest.repos.get(context.repo) | |
| return repo.data | |
| - name: Calculate output values | |
| id: prep | |
| run: | | |
| VERSION=noop | |
| SEMVER="^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-((0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*))*))?(\+([0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*))?$" | |
| if [ "${{ github.event_name }}" = "schedule" ]; then | |
| VERSION=nightly | |
| elif [ "${{ github.event_name }}" = "workflow_dispatch" ]; then | |
| VERSION=${{ github.event.inputs.image-tag }} | |
| elif [[ $GITHUB_REF == refs/tags/* ]]; then | |
| VERSION=${GITHUB_REF#refs/tags/} | |
| elif [[ $GITHUB_REF == refs/heads/* ]]; then | |
| VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g') | |
| if [ "${{ github.event.repository.default_branch }}" = "$VERSION" ]; | |
| then | |
| VERSION=edge | |
| fi | |
| elif [[ $GITHUB_REF == refs/pull/* ]]; then | |
| VERSION=pr-${{ github.event.number }} | |
| fi | |
| if [[ $VERSION =~ $SEMVER ]]; then | |
| VERSION_NO_V=${VERSION#v} | |
| MAJOR="${BASH_REMATCH[1]}" | |
| MINOR="${BASH_REMATCH[2]}" | |
| PATCH="${BASH_REMATCH[3]}" | |
| TAGS="${IMAGE_NAME}:${VERSION_NO_V//+/_},${IMAGE_NAME}:${MAJOR}.${MINOR}.${PATCH},${IMAGE_NAME}:${MAJOR}.${MINOR},${IMAGE_NAME}:${MAJOR},${IMAGE_NAME}:latest" | |
| else | |
| TAGS="${IMAGE_NAME}:${VERSION}" | |
| fi | |
| if [ "${{ github.event_name }}" = "push" ]; then | |
| TAGS="${TAGS},${IMAGE_NAME}:sha-${GITHUB_SHA::8}" | |
| fi | |
| for i in ${TAGS//,/ } | |
| do | |
| TAGS="${TAGS},ghcr.io/${i}" | |
| done | |
| echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT | |
| echo "source_version=$(./bump_version.sh show)" >> $GITHUB_OUTPUT | |
| echo "tags=${TAGS}" >> $GITHUB_OUTPUT | |
| echo tags=${TAGS} | |
| - name: Setup tmate debug session | |
| uses: mxschmitt/action-tmate@v3 | |
| if: github.event.inputs.remote-shell == 'true' || env.RUN_TMATE | |
| build: | |
| # Builds a single test image for the native platform. This image is saved | |
| # as an artifact and loaded by the test job. | |
| name: Build test image | |
| needs: | |
| - diagnostics | |
| - prepare | |
| runs-on: ubuntu-latest | |
| steps: | |
| - id: harden-runner | |
| name: Harden the runner | |
| uses: step-security/harden-runner@v2 | |
| with: | |
| egress-policy: audit | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Cache Docker layers | |
| uses: actions/cache@v3 | |
| env: | |
| BASE_CACHE_KEY: buildx-${{ runner.os }}- | |
| with: | |
| path: ${{ env.BUILDX_CACHE_DIR }} | |
| key: ${{ env.BASE_CACHE_KEY }}${{ github.sha }} | |
| restore-keys: | | |
| ${{ env.BASE_CACHE_KEY }} | |
| - name: Create dist directory | |
| run: mkdir -p dist | |
| - name: Build image | |
| id: docker_build | |
| uses: docker/build-push-action@v5 | |
| with: | |
| build-args: | | |
| VERSION=${{ needs.prepare.outputs.source_version }} | |
| cache-from: type=local,src=${{ env.BUILDX_CACHE_DIR }} | |
| cache-to: type=local,dest=${{ env.BUILDX_CACHE_DIR }} | |
| context: . | |
| file: ./Dockerfile | |
| outputs: type=docker,dest=dist/image.tar | |
| # Uncomment the following option if you are building an image for use | |
| # on Google Cloud Run or AWS Lambda. The current default image output | |
| # is unable to run on either. Please see the following issue for more | |
| # information: https://github.com/docker/buildx/issues/1533 | |
| # provenance: false | |
| tags: ${{ env.IMAGE_NAME }}:latest # not to be pushed | |
| # For a list of pre-defined annotation keys and value types see: | |
| # https://github.com/opencontainers/image-spec/blob/master/annotations.md | |
| labels: "\ | |
| org.opencontainers.image.created=${{ | |
| needs.prepare.outputs.created }} | |
| org.opencontainers.image.description=${{ | |
| fromJson(needs.prepare.outputs.repometa).description }} | |
| org.opencontainers.image.licenses=${{ | |
| fromJson(needs.prepare.outputs.repometa).license.spdx_id }} | |
| org.opencontainers.image.revision=${{ github.sha }} | |
| org.opencontainers.image.source=${{ | |
| fromJson(needs.prepare.outputs.repometa).clone_url }} | |
| org.opencontainers.image.title=${{ | |
| fromJson(needs.prepare.outputs.repometa).name }} | |
| org.opencontainers.image.url=${{ | |
| fromJson(needs.prepare.outputs.repometa).html_url }} | |
| org.opencontainers.image.version=${{ | |
| needs.prepare.outputs.source_version }}" | |
| - name: Compress image | |
| run: gzip dist/image.tar | |
| - name: Upload artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: dist | |
| path: dist | |
| - name: Setup tmate debug session | |
| uses: mxschmitt/action-tmate@v3 | |
| if: env.RUN_TMATE | |
| test: | |
| # Executes tests on the single-platform image created in the "build" job. | |
| name: Test image | |
| needs: | |
| - diagnostics | |
| - build | |
| runs-on: ubuntu-latest | |
| steps: | |
| - id: harden-runner | |
| name: Harden the runner | |
| uses: step-security/harden-runner@v2 | |
| with: | |
| egress-policy: audit | |
| - uses: actions/checkout@v4 | |
| - id: setup-python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.11" | |
| - name: Cache testing environments | |
| uses: actions/cache@v3 | |
| env: | |
| BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ | |
| py${{ steps.setup-python.outputs.python-version }}-" | |
| with: | |
| path: ${{ env.PIP_CACHE_DIR }} | |
| key: "${{ env.BASE_CACHE_KEY }}\ | |
| ${{ hashFiles('**/requirements-test.txt') }}-\ | |
| ${{ hashFiles('**/requirements.txt') }}" | |
| restore-keys: | | |
| ${{ env.BASE_CACHE_KEY }} | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip setuptools wheel | |
| pip install --upgrade --requirement requirements-test.txt | |
| - name: Download docker image artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: dist | |
| path: dist | |
| - name: Load docker image | |
| run: docker load < dist/image.tar.gz | |
| - name: Run tests | |
| env: | |
| RELEASE_TAG: ${{ github.event.release.tag_name }} | |
| run: pytest --runslow | |
| - name: Setup tmate debug session | |
| uses: mxschmitt/action-tmate@v3 | |
| if: env.RUN_TMATE | |
| build-push-all: | |
| # Builds the final set of images for each of the platforms listed in | |
| # PLATFORMS environment variable. These images are tagged with the Docker | |
| # tags calculated in the "prepare" job and pushed to Docker Hub and the | |
| # GitHub Container Registry. The contents of README.md are pushed as the | |
| # image's description to Docker Hub. This job is skipped when the | |
| # triggering event is a pull request. | |
| if: github.event_name != 'pull_request' | |
| name: Build and push all platforms | |
| needs: | |
| - diagnostics | |
| - lint | |
| - prepare | |
| - test | |
| # When Dependabot creates a PR it requires this permission in | |
| # order to push Docker images to ghcr.io. | |
| permissions: | |
| packages: write | |
| runs-on: ubuntu-latest | |
| steps: | |
| - id: harden-runner | |
| name: Harden the runner | |
| uses: step-security/harden-runner@v2 | |
| with: | |
| egress-policy: audit | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Cache Docker layers | |
| uses: actions/cache@v3 | |
| env: | |
| BASE_CACHE_KEY: buildx-${{ runner.os }}- | |
| with: | |
| path: ${{ env.BUILDX_CACHE_DIR }} | |
| key: ${{ env.BASE_CACHE_KEY }}${{ github.sha }} | |
| restore-keys: | | |
| ${{ env.BASE_CACHE_KEY }} | |
| - name: Create cross-platform support Dockerfile-x | |
| run: ./buildx-dockerfile.sh | |
| - name: Build and push platform images to registries | |
| id: docker_build | |
| uses: docker/build-push-action@v5 | |
| with: | |
| build-args: | | |
| VERSION=${{ needs.prepare.outputs.source_version }} | |
| cache-from: type=local,src=${{ env.BUILDX_CACHE_DIR }} | |
| cache-to: type=local,dest=${{ env.BUILDX_CACHE_DIR }} | |
| context: . | |
| file: ./Dockerfile-x | |
| platforms: ${{ env.PLATFORMS }} | |
| # Uncomment the following option if you are building an image for use | |
| # on Google Cloud Run or AWS Lambda. The current default image output | |
| # is unable to run on either. Please see the following issue for more | |
| # information: https://github.com/docker/buildx/issues/1533 | |
| # provenance: false | |
| push: true | |
| tags: ${{ needs.prepare.outputs.tags }} | |
| # For a list of pre-defined annotation keys and value types see: | |
| # https://github.com/opencontainers/image-spec/blob/master/annotations.md | |
| labels: "\ | |
| org.opencontainers.image.created=${{ | |
| needs.prepare.outputs.created }} | |
| org.opencontainers.image.description=${{ | |
| fromJson(needs.prepare.outputs.repometa).description }} | |
| org.opencontainers.image.licenses=${{ | |
| fromJson(needs.prepare.outputs.repometa).license.spdx_id }} | |
| org.opencontainers.image.revision=${{ github.sha }} | |
| org.opencontainers.image.source=${{ | |
| fromJson(needs.prepare.outputs.repometa).clone_url }} | |
| org.opencontainers.image.title=${{ | |
| fromJson(needs.prepare.outputs.repometa).name }} | |
| org.opencontainers.image.url=${{ | |
| fromJson(needs.prepare.outputs.repometa).html_url }} | |
| org.opencontainers.image.version=${{ | |
| needs.prepare.outputs.source_version }}" | |
| - name: Publish README.md to Docker Hub | |
| env: | |
| DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} | |
| DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} | |
| run: ./push_readme.sh | |
| - name: Setup tmate debug session | |
| uses: mxschmitt/action-tmate@v3 | |
| if: env.RUN_TMATE |