cloudbuild.yaml

#
#  Author: Hari Sekhon
#  Date: [% DATE # 2020-02-11 12:53:33 +0000 (Tue, 11 Feb 2020) %]
#
#  vim:ts=2:sts=2:sw=2:et
#
#  [% URL %]
#
#  [% LICENSE %]
#
#  [% MESSAGE %]
#
#  [% LINKEDIN %]
#

# ============================================================================ #
#                         G C P   C l o u d   B u i l d
# ============================================================================ #

# References:
#
#     https://cloud.google.com/cloud-build/docs/build-config
#
#     https://cloud.google.com/cloud-build/docs/speeding-up-builds
#
#     https://cloud.google.com/cloud-build/docs/building-leaner-containers
#
#     https://cloud.google.com/cloud-build/docs/build-debug-locally
#
#     https://cloud.google.com/cloud-build/docs/access-private-github-repos
#

# gcloud builds submit  # --config cloudbuild.yaml .

# cloud-build-local --dryrun=false .  # --push --config cloudbuild.yaml

# Notes:
#
# - $PROJECT_ID is auto-populated by Cloud Build
# - tars $PWD to bucket called ${PROJECT_ID}_cloudbuild
# - $PWD is unpacked to /workspace inside the docker images being run to create builds

---
# https://cloud.google.com/build/docs/api/reference/rest/v1/projects.builds#buildoptions
options:
  dynamic_substitutions: true  # required for composing substitutions from other substitutions + eval'ing them to values in steps shell commands

  # workaround for substitutions only used to construct other substitution variables, avoiding this type of error:
  #
  #   ERROR: (gcloud.builds.submit) INVALID_ARGUMENT: generic::invalid_argument: key "_IMAGE_NAME" in the substitution data is not matched in the template;key "_PROJECT" in the substitution data is not matched in the template;key "_REGISTRY" in the substitution data is not matched in the template
  #
  # but this disables all variable misuse protections, so better to just fabricate a usage of the construction variables instead, see images section below
  #substitution_option: ALLOW_LOOSE
  #
  # https://cloud.google.com/build/docs/api/reference/rest/v1/projects.builds#machinetype
  #machineType: E2_HIGHCPU_8  # defaults to 1 CPU machine
  #diskSizeGb: 50  # up to 1000 GB
  #
  env:
    - DOCKER_BUILDKIT=1

# variables - can override these on the CLI via:
#
#   --substitutions=_IMAGE_NAME="myapp",_IMAGE_VERSION:"1.1"
#
substitutions:
  # XXX: set image and tag
  _IMAGE_NAME: myapp
  # requires string, not float
  _IMAGE_VERSION: version-undefined  # updating this may not bust the cache because we --cache-from multiple tags, you'd need to comment out the --cache-from or use --no-cache for a proper cache bust behaviour
  _GIT_BRANCH: branch-undefined
  _GIT_TAG: tag-undefined

  # Google Container Registry (legacy):
  #_REGISTRY: eu.gcr.io
  #_REGISTRY: gcr.io

  # Google Artifact Registry (new):
  _REGISTRY: europe-docker.pkg.dev     # multi-region
  #_REGISTRY: us-docker.pkg.dev        # multi-region
  #_REGISTRY: <region>-docker.pkg.dev  # regional
  _REPOSITORY: myapp  # XXX: Edit

  # XXX: if you hardcode _PROJECT to a shared project where you keep all your company images, then
  #      you'll need to switch your GCloud SDK core.project to submit to that project to get the right GCS permissions to store the image
  #
  # XXX: if debugging in a local builder $PROJECT_ID isn't set and comes out literally, breaking the build with the below error
  #      so in that case you must hardcode _PROJECT's value to get it build in local debug builder
  #
  #      invalid argument "gcr.io/$PROJECT_ID/myimage:latest" for "-t, --tag" flag: invalid reference format: repository name must be lowercase
  #
  _PROJECT: $PROJECT_ID
  # GKE details
  #_CLOUDSDK_CONTAINER_CLUSTER: mycluster
  #_CLOUDSDK_COMPUTE_ZONE: europe-west1-b

  # XXX: massively shortens step commands
  #
  # For GCR:
  #_IMAGE: ${_REGISTRY}/${_PROJECT}/${_IMAGE_NAME}
  #
  # For Artifact Registry:
  _IMAGE: ${_REGISTRY}/${_PROJECT}/${_REPOSITORY}/${_IMAGE_NAME}

timeout: 3660s

# push this image to GCR - doing it this way instead of docker push links the image artifact in GCP Console's Cloud Build UI
images:
  # XXX: fabricated usage of substitution variables to avoid error described above at 'substitution_option: ALLOW_LOOSE'
  #- ${_IMAGE}
  - ${_REGISTRY}/${_PROJECT}/${_IMAGE_NAME}
  # the above pushed all tags to GCR without specifying tags explicitly like this:
  #- ${_IMAGE}:latest
  #- ${_IMAGE}:${_IMAGE_VERSION}

# Usually this isn't needed because cloudbuild tarballs and uploads $PWD - this is to access another service using an API token, SSH key etc.
# XXX: must grant "Secret Manager Secret Accessor" to ...@cloudbuild.gserviceaccount.com
availableSecrets:
  secretManager:
    - versionName: projects/${_PROJECT}/secrets/SECRET-NAME/versions/latest  # XXX: replace SECRET-NAME
      env: 'SSH_KEY'
      # XXX: add to steps below under
      #
      # secretEnv:
      #   - SSH_KEY

steps:
  # Debug build environment, find we run inside $PWD of /workspace where our local $PWD was unpacked
  #- name: busybox
  #  args:
  #    - sh
  #    - -c
  #    - pwd; ls -l
  #
  # ============================================================================ #
  #                              P u l l   C a c h e
  # ============================================================================ #
  # OLD: don't need to do this any more on newere versions of Docker
  # Pull previous builds to re-use their cache layers
  # use Docker cloud builder to pull image - not available in eu.gcr.io
  #- id: pull-image-cache-latest
  #  name: gcr.io/cloud-builders/docker
  #  waitFor: ['-']  # don't wait for any other steps
  #  entrypoint: 'bash'
  #  args:
  #    - -c
  #    - docker pull ${_IMAGE}:latest || exit 0
  ##
  #- id: pull-image-cache-version
  #  name: gcr.io/cloud-builders/docker
  #  waitFor: ['-']  # don't wait for any other steps
  #  entrypoint: 'bash'
  #  args:
  #    - -c
  #    - docker pull ${_IMAGE}:${_IMAGE_VERSION} || exit 0
  # ============================================================================ #
  #                             B u i l d   I m a g e
  # ============================================================================ #
  # use Docker cloud builder to build image - not available in eu.gcr.io
  - id: build-myimage  # optional but useful in the logs, otherwise you just get Step #1, Step #2 prefixes which make it difficult to differentiate which step an error comes from
    name: gcr.io/cloud-builders/docker
    env:
      - MYKEY=myvalue
    secretEnv:
      - SSH_KEY  # from 'availableSecrets: secretManager: ...' section above
    # multiple tags (all tags get pushed to GCR implicitly via image declaration above)
    args:
      - build
      # extra variables available for builds invoked by triggers:
      # https://cloud.google.com/cloud-build/docs/configuring-builds/substitute-variable-values
      - --tag=${_IMAGE}:latest
      - --tag=${_IMAGE}:${_IMAGE_VERSION}
      # use all cached layers where possible (will pull, better for seldomly rebuilt images)
      - --build-arg=BUILDKIT_INLINE_CACHE=1
      - --cache-from=${_IMAGE}:latest
      - --cache-from=${_IMAGE}:${_IMAGE_VERSION}
      - --cache-from=${_IMAGE}:${_GIT_BRANCH}
      - --cache-from=${_IMAGE}:${_GIT_TAG}
      - --cache-from=${_IMAGE}:staging  # explicitly try from staging and production branche latest too because reverts in lower environments will match these and reuse them as cache
      - --cache-from=${_IMAGE}:production
      #
      # Multi-stage layer caching - CloudBuild still doesn't support --cache-to as of Jan 2022 (GitHub Actions -> Docker action does, see my GitHub-Actions repo's reusable Docker workflows)
      #- --cache-to=type=registry,ref=${_IMAGE}:buildcache_${_GIT_BRANCH}_${_GIT_TAG},mode=max
      #- --tag=${_IMAGE}:buildcache_${_GIT_BRANCH}_${_GIT_TAG}
      #- --cache-from=${_IMAGE}:buildcache_${_GIT_BRANCH}_${_GIT_TAG}
      #
      # Specifying a different Dockerfile path is tricky - requires /workspace prefix and cannot be quoted or have a space between args otherwise ends up with this error trying to lstat blank or quote as first dir component:
      #
      #   failed to solve with frontend dockerfile.v0: failed to read dockerfile: error from sender: failed to resolve : lstat  : no such file or directory
      #
      #- --file=/workspace/DIR1/DIR2/Dockerfile
      #  this works too:
      #- -f=/workspace/DIR1/DIR2/Dockerfile
      - .
    # or use kaniko userspace builder without docker - https://github.com/GoogleContainerTools/kaniko
    #name: '${_REGISTRY}/kaniko-project/executor:latest'
    #args:
    #  - --destination=${_IMAGE}:${_IMAGE_VERSION}
    #  - --cache=true
    #  - --cache-ttl=6h
    timeout: 3600s

  # check the newly created image with a test command
  - name: '${_IMAGE}:${_IMAGE_VERSION}'
    args:
      - test
    timeout: 60s

  #- name: '${_REGISTRY}/cloud-builders/docker'
  #  args: ['push', '${_IMAGE}:${_IMAGE_VERSION}']

  # CI/CD - deploy to GKE
  #
  #- name: '${_REGISTRY}/cloud-builders/kubectl'
  #  args: ['set', 'image', 'deployment/my-deployment', 'my-container=${_IMAGE}:${_IMAGE_VERSION}']
  #  env:
  #  - 'CLOUDSDK_COMPUTE_ZONE=${_CLOUDSDK_COMPUTE_ZONE}'
  #  - 'CLOUDSDK_CONTAINER_CLUSTER=${_CLOUDSDK_CONTAINER_CLUSTER}'
  #  timeout: 600s

# not used in builds - only used for filtering build jobs in GCP CloudBuild UI  - https://cloud.google.com/build/docs/view-build-results#filtering_build_results_using_tags
tags:
  - build
  #- myapp
  - ${_IMAGE_NAME}