fix(docker): Prevent all possible "silent errors" during docker build

.dockerignore

@@ -2,3 +2,4 @@
 !.dockerignore
 !Dockerfile
 !tools/entrypoint.sh
+!tools/install/*.sh

.github/workflows/build-image-test.yaml

@@ -27,6 +27,13 @@ jobs:
             .dockerignore
             tools/entrypoint.sh
             .github/workflows/build-image-test.yaml
+            tools/*.sh
+
+      - name: Set up QEMU
+        if: matrix.os != 'ubuntu-latest' || matrix.arch != 'amd64'
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: 'arm64'
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
@@ -38,7 +45,7 @@ jobs:
           context: .
           build-args: |
             INSTALL_ALL=true
-          platforms: linux/amd64 # Only one allowed here, see https://github.com/docker/buildx/issues/59#issuecomment-1433097926
+          platforms: linux/${{ matrix.arch }} # Only one allowed here, see https://github.com/docker/buildx/issues/59#issuecomment-1433097926
           push: false
           load: true
           tags: |

.pre-commit-config.yaml

@@ -42,13 +42,14 @@ repos:
   hooks:
     - id: hadolint
       args: [
-        '--ignore', 'DL3027', # Do not use apt
         '--ignore', 'DL3007', # Using latest
+        '--ignore', 'DL3013', # Pin versions in pip
+        '--ignore', 'DL3027', # Do not use apt
+        '--ignore', 'DL3059', # Docker `RUN`s shouldn't be consolidated here
         '--ignore', 'DL4006', # Not related to alpine
         '--ignore', 'SC1091', # Useless check
         '--ignore', 'SC2015', # Useless check
         '--ignore', 'SC3037', # Not related to alpine
-        '--ignore', 'DL3013', # Pin versions in pip
       ]
 
 # JSON5 Linter

Dockerfile

@@ -7,209 +7,87 @@ WORKDIR /bin_dir
 
 RUN apk add --no-cache \
     # Builder deps
+    bash=~5 \
     curl=~8 && \
     # Upgrade packages for be able get latest Checkov
     python3 -m pip install --no-cache-dir --upgrade \
         pip \
         setuptools
 
+COPY tools/install/ /install/
+
+#
+# Install required tools
+#
 ARG PRE_COMMIT_VERSION=${PRE_COMMIT_VERSION:-latest}
 ARG TERRAFORM_VERSION=${TERRAFORM_VERSION:-latest}
 
-# Install pre-commit
-RUN if [ ${PRE_COMMIT_VERSION} = "latest" ]; \
-        then pip3 install --no-cache-dir pre-commit; \
-        else pip3 install --no-cache-dir pre-commit==${PRE_COMMIT_VERSION}; \
-    fi
+RUN touch /.env && \
+    if [ "$PRE_COMMIT_VERSION" = "false" ]; then echo "PRE_COMMIT_VERSION=latest" >> /.env; fi; \
+    if [ "$TERRAFORM_VERSION" = "false" ]; then echo "TERRAFORM_VERSION=latest" >> /.env; fi
 
-# Install terraform because pre-commit needs it
-RUN if [ "${TERRAFORM_VERSION}" = "latest" ]; then \
-        TERRAFORM_VERSION="$(curl -s https://api.github.com/repos/hashicorp/terraform/releases/latest | grep tag_name | grep -o -E -m 1 "[0-9.]+")" \
-    ; fi && \
-    curl -L "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_${TARGETOS}_${TARGETARCH}.zip" > terraform.zip && \
-    unzip terraform.zip terraform && rm terraform.zip
+RUN /install/pre-commit.sh
+RUN /install/terraform.sh
 
 #
 # Install tools
 #
 ARG CHECKOV_VERSION=${CHECKOV_VERSION:-false}
+ARG HCLEDIT_VERSION=${HCLEDIT_VERSION:-false}
 ARG INFRACOST_VERSION=${INFRACOST_VERSION:-false}
 ARG TERRAFORM_DOCS_VERSION=${TERRAFORM_DOCS_VERSION:-false}
 ARG TERRAGRUNT_VERSION=${TERRAGRUNT_VERSION:-false}
 ARG TERRASCAN_VERSION=${TERRASCAN_VERSION:-false}
 ARG TFLINT_VERSION=${TFLINT_VERSION:-false}
 ARG TFSEC_VERSION=${TFSEC_VERSION:-false}
-ARG TRIVY_VERSION=${TRIVY_VERSION:-false}
 ARG TFUPDATE_VERSION=${TFUPDATE_VERSION:-false}
-ARG HCLEDIT_VERSION=${HCLEDIT_VERSION:-false}
+ARG TRIVY_VERSION=${TRIVY_VERSION:-false}
 
 
 # Tricky thing to install all tools by set only one arg.
 # In RUN command below used `. /.env` <- this is sourcing vars that
 # specified in step below
 ARG INSTALL_ALL=${INSTALL_ALL:-false}
 RUN if [ "$INSTALL_ALL" != "false" ]; then \
-        echo "export CHECKOV_VERSION=latest" >> /.env && \
-        echo "export INFRACOST_VERSION=latest" >> /.env && \
-        echo "export TERRAFORM_DOCS_VERSION=latest" >> /.env && \
-        echo "export TERRAGRUNT_VERSION=latest" >> /.env && \
-        echo "export TERRASCAN_VERSION=latest" >> /.env && \
-        echo "export TFLINT_VERSION=latest" >> /.env && \
-        echo "export TFSEC_VERSION=latest" >> /.env && \
-        echo "export TRIVY_VERSION=latest" >> /.env && \
-        echo "export TFUPDATE_VERSION=latest" >> /.env && \
-        echo "export HCLEDIT_VERSION=latest" >> /.env \
-    ; else \
-        touch /.env \
-    ; fi
-
-
-# Checkov
-RUN . /.env && \
-    if [ "$CHECKOV_VERSION" != "false" ]; then \
-    ( \
-        # cargo, gcc, git, musl-dev, rust and CARGO envvar required for compilation of [email protected], no longer required once checkov version depends on rustworkx >0.14.0
-        # https://github.com/bridgecrewio/checkov/pull/6045
-        # gcc libffi-dev musl-dev required for compilation of cffi, until it contains musl aarch64
-        export CARGO_NET_GIT_FETCH_WITH_CLI=true && \
-        apk add --no-cache cargo=~1 gcc=~12 git=~2 libffi-dev=~3 libgcc=~12 musl-dev=~1 rust=~1 ; \
-        if [ "$CHECKOV_VERSION" = "latest" ]; \
-            then pip3 install --no-cache-dir checkov || exit 1; \
-            else pip3 install --no-cache-dir checkov==${CHECKOV_VERSION} || exit 1; \
-        fi; \
-        apk del cargo gcc git libffi-dev musl-dev rust \
-    ) \
-    ; fi
-
-# infracost
-RUN . /.env && \
-    if [ "$INFRACOST_VERSION" != "false" ]; then \
-    ( \
-        INFRACOST_RELEASES="https://api.github.com/repos/infracost/infracost/releases" && \
-        if [ "$INFRACOST_VERSION" = "latest" ]; \
-            then curl -L "$(curl -s ${INFRACOST_RELEASES}/latest | grep -o -E -m 1 "https://.+?-${TARGETOS}-${TARGETARCH}.tar.gz")" > infracost.tgz; \
-            else curl -L "$(curl -s ${INFRACOST_RELEASES} | grep -o -E "https://.+?v${INFRACOST_VERSION}/infracost-${TARGETOS}-${TARGETARCH}.tar.gz")" > infracost.tgz; \
-        fi; \
-    ) && tar -xzf infracost.tgz && rm infracost.tgz && mv infracost-${TARGETOS}-${TARGETARCH} infracost \
-    ; fi
-
-# Terraform docs
-RUN . /.env && \
-    if [ "$TERRAFORM_DOCS_VERSION" != "false" ]; then \
-    ( \
-        TERRAFORM_DOCS_RELEASES="https://api.github.com/repos/terraform-docs/terraform-docs/releases" && \
-        if [ "$TERRAFORM_DOCS_VERSION" = "latest" ]; \
-            then curl -L "$(curl -s ${TERRAFORM_DOCS_RELEASES}/latest | grep -o -E -m 1 "https://.+?-${TARGETOS}-${TARGETARCH}.tar.gz")" > terraform-docs.tgz; \
-            else curl -L "$(curl -s ${TERRAFORM_DOCS_RELEASES} | grep -o -E "https://.+?v${TERRAFORM_DOCS_VERSION}-${TARGETOS}-${TARGETARCH}.tar.gz")" > terraform-docs.tgz; \
-        fi; \
-    ) && tar -xzf terraform-docs.tgz terraform-docs && rm terraform-docs.tgz && chmod +x terraform-docs \
-    ; fi
-
-# Terragrunt
-RUN . /.env \
-    && if [ "$TERRAGRUNT_VERSION" != "false" ]; then \
-    ( \
-        TERRAGRUNT_RELEASES="https://api.github.com/repos/gruntwork-io/terragrunt/releases" && \
-        if [ "$TERRAGRUNT_VERSION" = "latest" ]; \
-            then curl -L "$(curl -s ${TERRAGRUNT_RELEASES}/latest | grep -o -E -m 1 "https://.+?/terragrunt_${TARGETOS}_${TARGETARCH}")" > terragrunt; \
-            else curl -L "$(curl -s ${TERRAGRUNT_RELEASES} | grep -o -E -m 1 "https://.+?v${TERRAGRUNT_VERSION}/terragrunt_${TARGETOS}_${TARGETARCH}")" > terragrunt; \
-        fi; \
-    ) && chmod +x terragrunt \
-    ; fi
-
-
-# Terrascan
-RUN . /.env && \
-    if [ "$TERRASCAN_VERSION" != "false" ]; then \
-    if [ "$TARGETARCH" != "amd64" ]; then ARCH="$TARGETARCH"; else ARCH="x86_64"; fi; \
-    # Convert the first letter to Uppercase
-    OS="$(echo ${TARGETOS} | cut -c1 | tr '[:lower:]' '[:upper:]' | xargs echo -n; echo ${TARGETOS} | cut -c2-)"; \
-    ( \
-        TERRASCAN_RELEASES="https://api.github.com/repos/tenable/terrascan/releases" && \
-        if [ "$TERRASCAN_VERSION" = "latest" ]; \
-            then curl -L "$(curl -s ${TERRASCAN_RELEASES}/latest | grep -o -E -m 1 "https://.+?_${OS}_${ARCH}.tar.gz")" > terrascan.tar.gz; \
-            else curl -L "$(curl -s ${TERRASCAN_RELEASES} | grep -o -E "https://.+?${TERRASCAN_VERSION}_${OS}_${ARCH}.tar.gz")" > terrascan.tar.gz; \
-        fi; \
-    ) && tar -xzf terrascan.tar.gz terrascan && rm terrascan.tar.gz && \
-    ./terrascan init \
-    ; fi
-
-# TFLint
-RUN . /.env && \
-    if [ "$TFLINT_VERSION" != "false" ]; then \
-    ( \
-        TFLINT_RELEASES="https://api.github.com/repos/terraform-linters/tflint/releases" && \
-        if [ "$TFLINT_VERSION" = "latest" ]; \
-            then curl -L "$(curl -s ${TFLINT_RELEASES}/latest | grep -o -E -m 1 "https://.+?_${TARGETOS}_${TARGETARCH}.zip")" > tflint.zip; \
-            else curl -L "$(curl -s ${TFLINT_RELEASES} | grep -o -E "https://.+?/v${TFLINT_VERSION}/tflint_${TARGETOS}_${TARGETARCH}.zip")" > tflint.zip; \
-        fi; \
-    ) && unzip tflint.zip && rm tflint.zip \
-    ; fi
-
-# TFSec
-RUN . /.env && \
-    if [ "$TFSEC_VERSION" != "false" ]; then \
-    ( \
-        TFSEC_RELEASES="https://api.github.com/repos/aquasecurity/tfsec/releases" && \
-        if [ "$TFSEC_VERSION" = "latest" ]; then \
-            curl -L "$(curl -s ${TFSEC_RELEASES}/latest | grep -o -E -m 1 "https://.+?/tfsec-${TARGETOS}-${TARGETARCH}")" > tfsec; \
-            else curl -L "$(curl -s ${TFSEC_RELEASES} | grep -o -E -m 1 "https://.+?v${TFSEC_VERSION}/tfsec-${TARGETOS}-${TARGETARCH}")" > tfsec; \
-        fi; \
-    ) && chmod +x tfsec \
+        echo "CHECKOV_VERSION=latest"        >> /.env && \
+        echo "INFRACOST_VERSION=latest"      >> /.env && \
+        echo "TERRAFORM_DOCS_VERSION=latest" >> /.env && \
+        echo "TERRAGRUNT_VERSION=latest"     >> /.env && \
+        echo "TERRASCAN_VERSION=latest"      >> /.env && \
+        echo "TFLINT_VERSION=latest"         >> /.env && \
+        echo "TFSEC_VERSION=latest"          >> /.env && \
+        echo "TRIVY_VERSION=latest"          >> /.env && \
+        echo "TFUPDATE_VERSION=latest"       >> /.env && \
+        echo "HCLEDIT_VERSION=latest"        >> /.env \
     ; fi
 
-# Trivy
-RUN . /.env && \
-    if [ "$TRIVY_VERSION" != "false" ]; then \
-    if [ "$TARGETARCH" != "amd64" ]; then ARCH="$TARGETARCH"; else ARCH="64bit"; fi; \
-    ( \
-        TRIVY_RELEASES="https://api.github.com/repos/aquasecurity/trivy/releases" && \
-        if [ "$TRIVY_VERSION" = "latest" ]; \
-            then curl -L "$(curl -s ${TRIVY_RELEASES}/latest | grep -o -E -i -m 1 "https://.+?/trivy_.+?_${TARGETOS}-${ARCH}.tar.gz")" > trivy.tar.gz; \
-            else curl -L "$(curl -s ${TRIVY_RELEASES} | grep -o -E -i -m 1 "https://.+?/v${TRIVY_VERSION}/trivy_.+?_${TARGETOS}-${ARCH}.tar.gz")" > trivy.tar.gz; \
-        fi; \
-    ) && tar -xzf trivy.tar.gz trivy && rm trivy.tar.gz \
-    ; fi
-
-# TFUpdate
-RUN . /.env && \
-    if [ "$TFUPDATE_VERSION" != "false" ]; then \
-    ( \
-        TFUPDATE_RELEASES="https://api.github.com/repos/minamijoyo/tfupdate/releases" && \
-        if [ "$TFUPDATE_VERSION" = "latest" ]; \
-            then curl -L "$(curl -s ${TFUPDATE_RELEASES}/latest | grep -o -E -m 1 "https://.+?_${TARGETOS}_${TARGETARCH}.tar.gz")" > tfupdate.tgz; \
-            else curl -L "$(curl -s ${TFUPDATE_RELEASES} | grep -o -E -m 1 "https://.+?${TFUPDATE_VERSION}_${TARGETOS}_${TARGETARCH}.tar.gz")" > tfupdate.tgz; \
-        fi; \
-    ) && tar -xzf tfupdate.tgz tfupdate && rm tfupdate.tgz \
-    ; fi
+RUN /install/checkov.sh
+RUN /install/hcledit.sh
+RUN /install/infracost.sh
+RUN /install/terraform-docs.sh
+RUN /install/terragrunt.sh
+RUN /install/terrascan.sh
+RUN /install/tflint.sh
+RUN /install/tfsec.sh
+RUN /install/tfupdate.sh
+RUN /install/trivy.sh
 
-# hcledit
-RUN . /.env && \
-    if [ "$HCLEDIT_VERSION" != "false" ]; then \
-    ( \
-        HCLEDIT_RELEASES="https://api.github.com/repos/minamijoyo/hcledit/releases" && \
-        if [ "$HCLEDIT_VERSION" = "latest" ]; \
-            then curl -L "$(curl -s ${HCLEDIT_RELEASES}/latest | grep -o -E -m 1 "https://.+?_${TARGETOS}_${TARGETARCH}.tar.gz")" > hcledit.tgz; \
-            else curl -L "$(curl -s ${HCLEDIT_RELEASES} | grep -o -E -m 1 "https://.+?${HCLEDIT_VERSION}_${TARGETOS}_${TARGETARCH}.tar.gz")" > hcledit.tgz; \
-        fi; \
-    ) && tar -xzf hcledit.tgz hcledit && rm hcledit.tgz \
-    ; fi
 
 # Checking binaries versions and write it to debug file
 RUN . /.env && \
     F=tools_versions_info && \
     pre-commit --version >> $F && \
     ./terraform --version | head -n 1 >> $F && \
     (if [ "$CHECKOV_VERSION"        != "false" ]; then echo "checkov $(checkov --version)" >> $F;     else echo "checkov SKIPPED" >> $F        ; fi) && \
+    (if [ "$HCLEDIT_VERSION"        != "false" ]; then echo "hcledit $(./hcledit version)" >> $F;     else echo "hcledit SKIPPED" >> $F        ; fi) && \
     (if [ "$INFRACOST_VERSION"      != "false" ]; then echo "$(./infracost --version)" >> $F;         else echo "infracost SKIPPED" >> $F      ; fi) && \
     (if [ "$TERRAFORM_DOCS_VERSION" != "false" ]; then ./terraform-docs --version >> $F;              else echo "terraform-docs SKIPPED" >> $F ; fi) && \
     (if [ "$TERRAGRUNT_VERSION"     != "false" ]; then ./terragrunt --version >> $F;                  else echo "terragrunt SKIPPED" >> $F     ; fi) && \
     (if [ "$TERRASCAN_VERSION"      != "false" ]; then echo "terrascan $(./terrascan version)" >> $F; else echo "terrascan SKIPPED" >> $F      ; fi) && \
     (if [ "$TFLINT_VERSION"         != "false" ]; then ./tflint --version >> $F;                      else echo "tflint SKIPPED" >> $F         ; fi) && \
     (if [ "$TFSEC_VERSION"          != "false" ]; then echo "tfsec $(./tfsec --version)" >> $F;       else echo "tfsec SKIPPED" >> $F          ; fi) && \
-    (if [ "$TRIVY_VERSION"          != "false" ]; then echo "trivy $(./trivy --version)" >> $F;       else echo "trivy SKIPPED" >> $F          ; fi) && \
     (if [ "$TFUPDATE_VERSION"       != "false" ]; then echo "tfupdate $(./tfupdate --version)" >> $F; else echo "tfupdate SKIPPED" >> $F       ; fi) && \
-    (if [ "$HCLEDIT_VERSION"        != "false" ]; then echo "hcledit $(./hcledit version)" >> $F;     else echo "hcledit SKIPPED" >> $F       ; fi) && \
+    (if [ "$TRIVY_VERSION"          != "false" ]; then echo "trivy $(./trivy --version)" >> $F;       else echo "trivy SKIPPED" >> $F          ; fi) && \
     echo -e "\n\n" && cat $F && echo -e "\n\n"
 
 

tools/install/_common.sh

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+# Tool name, based on filename.
+# Tool filename MUST BE same as in package manager/binary name
+TOOL=${0##*/}
+readonly TOOL=${TOOL%%.*}
+
+# Get "TOOL_VERSION"
+# shellcheck disable=SC1091 # Created in Dockerfile before execution of this script
+source /.env
+env_var_name="${TOOL//-/_}"
+env_var_name="${env_var_name^^}_VERSION"
+# shellcheck disable=SC2034 # Used in other scripts
+readonly VERSION="${!env_var_name}"
+
+# Skip tool installation if the version is set to "false"
+if [[ $VERSION == false ]]; then
+  echo "'$TOOL' skipped"
+  exit 0
+fi
+
+#######################################################################
+# Install the latest or specific version of the tool from GitHub release
+# Globals:
+#   TOOL - Name of the tool
+#   VERSION - Version of the tool
+# Arguments:
+#   GH_ORG - GitHub organization name where the tool is hosted
+#   DISTRIBUTED_AS - How the tool is distributed.
+#     Can be: 'tar.gz', 'zip' or 'binary'
+#   GH_RELEASE_REGEX_LATEST - Regular expression to match the latest
+#     release URL
+#   GH_RELEASE_REGEX_SPECIFIC_VERSION - Regular expression to match the
+#      specific version release URL
+#   UNUSUAL_TOOL_NAME_IN_PKG - If the tool in the tar.gz package is
+#     not in the root or named differently than the tool name itself,
+#     For example, includes the version number or is in a subdirectory
+#######################################################################
+function common::install_from_gh_release {
+  local -r GH_ORG=$1
+  local -r DISTRIBUTED_AS=$2
+  local -r GH_RELEASE_REGEX_LATEST=$3
+  local -r GH_RELEASE_REGEX_SPECIFIC_VERSION=$4
+  local -r UNUSUAL_TOOL_NAME_IN_PKG=$5
+
+  case $DISTRIBUTED_AS in
+    tar.gz | zip)
+      local -r PKG="${TOOL}.${DISTRIBUTED_AS}"
+      ;;
+    binary)
+      local -r PKG="$TOOL"
+      ;;
+    *)
+      echo "Unknown DISTRIBUTED_AS: '$DISTRIBUTED_AS'. Should be one of: 'tar.gz', 'zip' or 'binary'."
+      exit 1
+      ;;
+  esac
+
+  # Download tool
+  local -r RELEASES="https://api.github.com/repos/${GH_ORG}/${TOOL}/releases"
+
+  if [[ $VERSION == latest ]]; then
+    curl -L "$(curl -s "${RELEASES}/latest" | grep -o -E -i -m 1 "$GH_RELEASE_REGEX_LATEST")" > "$PKG"
+  else
+    curl -L "$(curl -s "$RELEASES" | grep -o -E -i -m 1 "$GH_RELEASE_REGEX_SPECIFIC_VERSION")" > "$PKG"
+  fi
+
+  # Make tool ready to use
+  if [[ $DISTRIBUTED_AS == tar.gz ]]; then
+    if [[ -z $UNUSUAL_TOOL_NAME_IN_PKG ]]; then
+      tar -xzf "$PKG" "$TOOL"
+    else
+      tar -xzf "$PKG" "$UNUSUAL_TOOL_NAME_IN_PKG"
+      mv "$UNUSUAL_TOOL_NAME_IN_PKG" "$TOOL"
+    fi
+    rm "$PKG"
+
+  elif [[ $DISTRIBUTED_AS == zip ]]; then
+    unzip "$PKG"
+    rm "$PKG"
+  else
+    chmod +x "$PKG"
+  fi
+}