From 81902924f1ce1a00bcadebcdb8871efe18526357 Mon Sep 17 00:00:00 2001 From: Dan Stefaniuk Date: Mon, 16 Oct 2023 07:23:49 +0100 Subject: [PATCH 1/5] Refactor shell scripts --- .tool-versions | 5 +++ scripts/docker/docker.lib.sh | 6 +-- scripts/docker/dockerfile-linter.sh | 2 +- scripts/docker/tests/docker.test.sh | 6 +-- scripts/githooks/check-file-format.sh | 44 +++++++++++++++---- scripts/githooks/check-markdown-format.sh | 50 ++++++++++++++++------ scripts/githooks/check-terraform-format.sh | 2 +- scripts/githooks/scan-secrets.sh | 6 ++- scripts/init.mk | 2 +- scripts/reports/cloc-repository.sh | 46 ++++++++++++++------ scripts/reports/generate-sbom.sh | 47 ++++++++++++++------ scripts/reports/perform-static-analysis.sh | 2 +- scripts/reports/scan-vulnerabilities.sh | 50 +++++++++++++++++----- scripts/shellscript-linter.sh | 2 +- scripts/terraform/terraform.lib.sh | 12 +++--- scripts/terraform/terraform.mk | 4 +- scripts/terraform/terraform.sh | 2 +- scripts/tests/unit.sh | 2 +- 18 files changed, 211 insertions(+), 79 deletions(-) diff --git a/.tool-versions b/.tool-versions index 2411b75e..2faaa205 100644 --- a/.tool-versions +++ b/.tool-versions @@ -7,9 +7,14 @@ pre-commit 3.4.0 # The section below is reserved for Docker image versions. # TODO: Move this section - consider using a different file for the repository template dependencies. +# docker/ghcr.io/anchore/grype v0.69.1@sha256:d41fcb371d0af59f311e72123dff46900ebd6d0482391b5a830853ee4f9d1a76 # SEE: https://github.com/anchore/grype/pkgs/container/grype +# docker/ghcr.io/anchore/syft v0.92.0@sha256:63c60f0a21efb13e80aa1359ab243e49213b6cc2d7e0f8179da38e6913b997e0 # SEE: https://github.com/anchore/syft/pkgs/container/syft # docker/ghcr.io/gitleaks/gitleaks v8.18.0@sha256:fd2b5cab12b563d2cc538b14631764a1c25577780e3b7dba71657d58da45d9d9 # SEE: https://github.com/gitleaks/gitleaks/pkgs/container/gitleaks +# docker/ghcr.io/igorshubovych/markdownlint-cli v0.37.0@sha256:fb3e79946fce78e1cde84d6798c6c2a55f2de11fc16606a40d49411e281d950d # SEE: https://github.com/igorshubovych/markdownlint-cli/pkgs/container/markdownlint-cli +# docker/ghcr.io/make-ops-tools/gocloc latest@sha256:6888e62e9ae693c4ebcfed9f1d86c70fd083868acb8815fe44b561b9a73b5032 # SEE: https://github.com/make-ops-tools/gocloc/pkgs/container/gocloc # docker/ghcr.io/nhs-england-tools/github-runner-image 20230909-321fd1e-rt@sha256:ce4fd6035dc450a50d3cbafb4986d60e77cb49a71ab60a053bb1b9518139a646 # SEE: https://github.com/nhs-england-tools/github-runner-image/pkgs/container/github-runner-image # docker/hadolint/hadolint 2.12.0-alpine@sha256:7dba9a9f1a0350f6d021fb2f6f88900998a4fb0aaf8e4330aa8c38544f04db42 # SEE: https://hub.docker.com/r/hadolint/hadolint/tags # docker/hashicorp/terraform 1.5.6@sha256:180a7efa983386a27b43657ed610e9deed9e6c3848d54f9ea9b6cb8a5c8c25f5 # SEE: https://hub.docker.com/r/hashicorp/terraform/tags # docker/koalaman/shellcheck latest@sha256:e40388688bae0fcffdddb7e4dea49b900c18933b452add0930654b2dea3e7d5c # SEE: https://hub.docker.com/r/koalaman/shellcheck/tags +# docker/mstruebing/editorconfig-checker 2.7.1@sha256:dd3ca9ea50ef4518efe9be018d669ef9cf937f6bb5cfe2ef84ff2a620b5ddc24 # SEE: https://hub.docker.com/r/mstruebing/editorconfig-checker/tags # docker/sonarsource/sonar-scanner-cli 5.0.1@sha256:494ecc3b5b1ee1625bd377b3905c4284e4f0cc155cff397805a244dee1c7d575 # SEE: https://hub.docker.com/r/sonarsource/sonar-scanner-cli/tags diff --git a/scripts/docker/docker.lib.sh b/scripts/docker/docker.lib.sh index 81f38c88..18787105 100644 --- a/scripts/docker/docker.lib.sh +++ b/scripts/docker/docker.lib.sh @@ -27,8 +27,8 @@ function docker-build() { version-create-effective-file _create-effective-dockerfile - # The current directory must be changed for the image build script to access - # assets that need to be copied + # The current directory must be changed for the image build script to access + # assets that need to be copied current_dir=$(pwd) cd "$dir" docker build \ @@ -164,7 +164,7 @@ function docker-get-image-version-and-pull() { # digest="sha256:hash" # Get the image full version from the '.tool-versions' file, - # match it by name and version regex, if given. + # match it by name and version regex, if given. local versions_file="${TOOL_VERSIONS:=$(git rev-parse --show-toplevel)/.tool-versions}" local version="latest" if [ -f "$versions_file" ]; then diff --git a/scripts/docker/dockerfile-linter.sh b/scripts/docker/dockerfile-linter.sh index 5929721d..9e74a8fb 100755 --- a/scripts/docker/dockerfile-linter.sh +++ b/scripts/docker/dockerfile-linter.sh @@ -12,8 +12,8 @@ set -euo pipefail # # Arguments (provided as environment variables): # file=Dockerfile # Path to the Dockerfile to lint, relative to the project's top-level directory, default is './Dockerfile.effective' -# VERBOSE=true # Show all the executed commands, default is 'false' # FORCE_USE_DOCKER=true # If set to true the command is run in a Docker container, default is 'false' +# VERBOSE=true # Show all the executed commands, default is 'false' # ============================================================================== diff --git a/scripts/docker/tests/docker.test.sh b/scripts/docker/tests/docker.test.sh index e2bd80cf..8f487b8f 100755 --- a/scripts/docker/tests/docker.test.sh +++ b/scripts/docker/tests/docker.test.sh @@ -73,12 +73,12 @@ function test-docker-build() { function test-docker-image-from-signature() { - # Arrange + # Arrange TOOL_VERSIONS="$(git rev-parse --show-toplevel)/scripts/docker/tests/.tool-versions.test" cp Dockerfile Dockerfile.effective - # Act + # Act _replace-image-latest-by-specific-version - # Assert + # Assert grep -q "FROM python:.*-alpine.*@sha256:.*" Dockerfile.effective && return 0 || return 1 } diff --git a/scripts/githooks/check-file-format.sh b/scripts/githooks/check-file-format.sh index d52139e9..75029641 100755 --- a/scripts/githooks/check-file-format.sh +++ b/scripts/githooks/check-file-format.sh @@ -6,13 +6,16 @@ set -euo pipefail # Pre-commit git hook to check the EditorConfig rules compliance over changed # files. It ensures all non-binary files across the codebase are formatted -# according to the style defined in the `.editorconfig` file. +# according to the style defined in the `.editorconfig` file. This is a +# editorconfig command wrapper. It will run editorconfig natively if it is +# installed, otherwise it will run it in a Docker container. # # Usage: # $ check={all,staged-changes,working-tree-changes,branch} [dry_run=true] ./check-file-format.sh # # Options: # BRANCH_NAME=other-branch-than-main # Branch to compare with, default is `origin/main` +# FORCE_USE_DOCKER=true # If set to true the command is run in a Docker container, default is 'false' # VERBOSE=true # Show all the executed commands, default is `false` # # Exit codes: @@ -38,12 +41,6 @@ set -euo pipefail # ============================================================================== -# SEE: https://hub.docker.com/r/mstruebing/editorconfig-checker/tags, use the `linux/amd64` os/arch -image_version=2.7.1@sha256:dd3ca9ea50ef4518efe9be018d669ef9cf937f6bb5cfe2ef84ff2a620b5ddc24 - -# ============================================================================== - - function main() { cd "$(git rev-parse --show-toplevel)" @@ -70,13 +67,42 @@ function main() { ;; esac + if command -v editorconfig > /dev/null 2>&1 && ! is-arg-true "${FORCE_USE_DOCKER:-false}"; then + filter="$filter" dry_run_opt="${dry_run_opt:-}" cli-run-editorconfig + else + filter="$filter" dry_run_opt="${dry_run_opt:-}" docker-run-editorconfig + fi +} + +# Run editorconfig natively. +# Arguments (provided as environment variables): +# dry_run_opt=[dry run option] +# filter=[filter for files to check] +function cli-run-editorconfig() { + + # shellcheck disable=SC2046,SC2086 + editorconfig \ + --exclude '.git/' $dry_run_opt $($filter) +} + +# Run editorconfig in a Docker container. +# Arguments (provided as environment variables): +# dry_run_opt=[dry run option] +# filter=[filter for files to check] +function docker-run-editorconfig() { + + # shellcheck disable=SC1091 + source ./scripts/docker/docker.lib.sh + + # shellcheck disable=SC2155 + local image=$(name=mstruebing/editorconfig-checker docker-get-image-version-and-pull) # We use /dev/null here as a backstop in case there are no files in the state # we choose. If the filter comes back empty, adding `/dev/null` onto it has # the effect of preventing `ec` from treating "no files" as "all the files". docker run --rm --platform linux/amd64 \ --volume "$PWD":/check \ - mstruebing/editorconfig-checker:$image_version \ - sh -c "ec --exclude '.git/' ${dry_run_opt:-} \$($filter) /dev/null" + "$image" \ + sh -c "ec --exclude '.git/' $dry_run_opt \$($filter) /dev/null" } # ============================================================================== diff --git a/scripts/githooks/check-markdown-format.sh b/scripts/githooks/check-markdown-format.sh index c1d66e91..311e3e3a 100755 --- a/scripts/githooks/check-markdown-format.sh +++ b/scripts/githooks/check-markdown-format.sh @@ -5,13 +5,16 @@ set -euo pipefail # Pre-commit git hook to check the Markdown file formatting rules compliance -# over changed files. +# over changed files. This is a markdownlint command wrapper. It will run +# markdownlint natively if it is installed, otherwise it will run it in a Docker +# container. # # Usage: # $ check={all,staged-changes,working-tree-changes,branch} ./check-markdown-format.sh # # Options: # BRANCH_NAME=other-branch-than-main # Branch to compare with, default is `origin/main` +# FORCE_USE_DOCKER=true # If set to true the command is run in a Docker container, default is 'false' # VERBOSE=true # Show all the executed commands, default is `false` # # Exit codes: @@ -27,11 +30,6 @@ set -euo pipefail # ============================================================================== -# SEE: https://github.com/igorshubovych/markdownlint-cli/pkgs/container/markdownlint-cli, use the `linux/amd64` os/arch -image_version=v0.37.0@sha256:fb3e79946fce78e1cde84d6798c6c2a55f2de11fc16606a40d49411e281d950d - -# ============================================================================== - function main() { cd "$(git rev-parse --show-toplevel)" @@ -53,15 +51,43 @@ function main() { esac if [ -n "$files" ]; then - # shellcheck disable=SC2086 - docker run --rm --platform linux/amd64 \ - --volume "$PWD":/workdir \ - ghcr.io/igorshubovych/markdownlint-cli:$image_version \ - $files \ - --config /workdir/scripts/config/markdownlint.yaml + if command -v markdownlint > /dev/null 2>&1 && ! is-arg-true "${FORCE_USE_DOCKER:-false}"; then + files="$files" cli-run-markdownlint + else + files="$files" docker-run-markdownlint + fi fi } +# Run markdownlint natively. +# Arguments (provided as environment variables): +# files=[files to check] +function cli-run-markdownlint() { + + # shellcheck disable=SC2086 + markdownlint \ + $files \ + --config "$PWD/scripts/config/markdownlint.yaml" +} + +# Run markdownlint in a Docker container. +# Arguments (provided as environment variables): +# files=[files to check] +function docker-run-markdownlint() { + + # shellcheck disable=SC1091 + source ./scripts/docker/docker.lib.sh + + # shellcheck disable=SC2155 + local image=$(name=ghcr.io/igorshubovych/markdownlint-cli docker-get-image-version-and-pull) + # shellcheck disable=SC2086 + docker run --rm --platform linux/amd64 \ + --volume "$PWD":/workdir \ + "$image" \ + $files \ + --config /workdir/scripts/config/markdownlint.yaml +} + # ============================================================================== function is-arg-true() { diff --git a/scripts/githooks/check-terraform-format.sh b/scripts/githooks/check-terraform-format.sh index ef28f09b..ec40f86d 100755 --- a/scripts/githooks/check-terraform-format.sh +++ b/scripts/githooks/check-terraform-format.sh @@ -11,8 +11,8 @@ set -euo pipefail # # Options: # check_only=true # Do not format, run check only, default is 'false' -# VERBOSE=true # Show all the executed commands, default is 'false' # FORCE_USE_DOCKER=true # If set to true the command is run in a Docker container, default is 'false' +# VERBOSE=true # Show all the executed commands, default is 'false' # ============================================================================== diff --git a/scripts/githooks/scan-secrets.sh b/scripts/githooks/scan-secrets.sh index 9e7f3dc6..e076fb87 100755 --- a/scripts/githooks/scan-secrets.sh +++ b/scripts/githooks/scan-secrets.sh @@ -4,15 +4,17 @@ set -euo pipefail -# Pre-commit git hook to scan for secrets hard-coded in the codebase. +# Pre-commit git hook to scan for secrets hard-coded in the codebase. This is a +# gitleaks command wrapper. It will run gitleaks natively if it is installed, +# otherwise it will run it in a Docker container. # # Usage: # $ ./scan-secrets.sh # # Options: # check={whole-history,last-commit,staged-changes} # Type of the check to run, default is 'staged-changes' -# VERBOSE=true # Show all the executed commands, default is 'false' # FORCE_USE_DOCKER=true # If set to true the command is run in a Docker container, default is 'false' +# VERBOSE=true # Show all the executed commands, default is 'false' # # Exit codes: # 0 - No leaks present diff --git a/scripts/init.mk b/scripts/init.mk index 5cb2bdf2..eca74413 100644 --- a/scripts/init.mk +++ b/scripts/init.mk @@ -88,7 +88,7 @@ else .SHELLFLAGS := -ce endif -# This script parses all the make target descriptions and renders the help output. +# This script parses all the make target descriptions and renders the help output. HELP_SCRIPT = \ \ use Text::Wrap; \ diff --git a/scripts/reports/cloc-repository.sh b/scripts/reports/cloc-repository.sh index 062b8352..c510d98f 100755 --- a/scripts/reports/cloc-repository.sh +++ b/scripts/reports/cloc-repository.sh @@ -4,19 +4,17 @@ set -euo pipefail -# Count lines of code of this repository. +# Count lines of code of this repository. This is a gocloc command wrapper. It +# will run gocloc natively if it is installed, otherwise it will run it in a +# Docker container. # # Usage: # $ ./cloc-repository.sh # # Options: -# VERBOSE=true # Show all the executed commands, default is `false` # BUILD_DATETIME=%Y-%m-%dT%H:%M:%S%z # Build datetime, default is `date -u +'%Y-%m-%dT%H:%M:%S%z'` - -# ============================================================================== - -# SEE: https://github.com/make-ops-tools/gocloc/pkgs/container/gocloc, use the `linux/amd64` os/arch -image_version=latest@sha256:6888e62e9ae693c4ebcfed9f1d86c70fd083868acb8815fe44b561b9a73b5032 +# FORCE_USE_DOCKER=true # If set to true the command is run in a Docker container, default is 'false' +# VERBOSE=true # Show all the executed commands, default is `false` # ============================================================================== @@ -28,20 +26,44 @@ function main() { enrich-report } +# Create the report. function create-report() { + if command -v gocloc > /dev/null 2>&1 && ! is-arg-true "${FORCE_USE_DOCKER:-false}"; then + cli-run-gocloc + else + docker-run-gocloc + fi + # shellcheck disable=SC2002 + cat cloc-report.tmp.json \ + | jq -r '["Language","files","blank","comment","code"],["--------"],(.languages[]|[.name,.files,.blank,.comment,.code]),["-----"],(.total|["TOTAL",.files,.blank,.comment,.code])|@tsv' \ + | sed 's/Plain Text/Plaintext/g' \ + | column -t +} + +# Run gocloc natively. +function cli-run-gocloc() { + + gocloc --output-type=json . > cloc-report.tmp.json +} + +# Run gocloc in a Docker container. +function docker-run-gocloc() { + + # shellcheck disable=SC1091 + source ./scripts/docker/docker.lib.sh + + # shellcheck disable=SC2155 + local image=$(name=ghcr.io/make-ops-tools/gocloc docker-get-image-version-and-pull) docker run --rm --platform linux/amd64 \ --volume "$PWD":/workdir \ - ghcr.io/make-ops-tools/gocloc:$image_version \ + "$image" \ --output-type=json \ . \ > cloc-report.tmp.json - if which jq > /dev/null && which column > /dev/null; then - # shellcheck disable=SC2002 - cat cloc-report.tmp.json | jq -r '["Language","files","blank","comment","code"],["--------"],(.languages[]|[.name,.files,.blank,.comment,.code]),["-----"],(.total|["TOTAL",.files,.blank,.comment,.code])|@tsv' | column -t - fi } +# Include additional information in the report. function enrich-report() { build_datetime=${BUILD_DATETIME:-$(date -u +'%Y-%m-%dT%H:%M:%S%z')} diff --git a/scripts/reports/generate-sbom.sh b/scripts/reports/generate-sbom.sh index 39fd0b5a..ed861d3b 100755 --- a/scripts/reports/generate-sbom.sh +++ b/scripts/reports/generate-sbom.sh @@ -5,19 +5,17 @@ set -euo pipefail # Script to generate SBOM (Software Bill of Materials) for the repository -# content and any artefact created by the CI/CD pipeline. +# content and any artefact created by the CI/CD pipeline. This is a syft command +# wrapper. It will run syft natively if it is installed, otherwise it will run +# it in a Docker container. # # Usage: # $ ./generate-sbom.sh # # Options: -# VERBOSE=true # Show all the executed commands, default is `false` # BUILD_DATETIME=%Y-%m-%dT%H:%M:%S%z # Build datetime, default is `date -u +'%Y-%m-%dT%H:%M:%S%z'` - -# ============================================================================== - -# SEE: https://github.com/anchore/syft/pkgs/container/syft, use the `linux/amd64` os/arch -image_version=v0.92.0@sha256:63c60f0a21efb13e80aa1359ab243e49213b6cc2d7e0f8179da38e6913b997e0 +# FORCE_USE_DOCKER=true # If set to true the command is run in a Docker container, default is 'false' +# VERBOSE=true # Show all the executed commands, default is `false` # ============================================================================== @@ -29,16 +27,41 @@ function main() { enrich-report } +# Create the report. function create-report() { + if command -v syft > /dev/null 2>&1 && ! is-arg-true "${FORCE_USE_DOCKER:-false}"; then + cli-run-syft + else + docker-run-syft + fi +} + +# Run syft natively. +function cli-run-syft() { + + syft packages dir:"$PWD" \ + --config "$PWD/scripts/config/syft.yaml" \ + --output spdx-json="$PWD/sbom-repository-report.tmp.json" +} + +# Run syft in a Docker container. +function docker-run-syft() { + + # shellcheck disable=SC1091 + source ./scripts/docker/docker.lib.sh + + # shellcheck disable=SC2155 + local image=$(name=ghcr.io/anchore/syft docker-get-image-version-and-pull) docker run --rm --platform linux/amd64 \ - --volume "$PWD":/scan \ - ghcr.io/anchore/syft:$image_version \ - packages dir:/scan \ - --config /scan/scripts/config/syft.yaml \ - --output spdx-json=/scan/sbom-repository-report.tmp.json + --volume "$PWD":/workdir \ + "$image" \ + packages dir:/workdir \ + --config /workdir/scripts/config/syft.yaml \ + --output spdx-json=/workdir/sbom-repository-report.tmp.json } +# Include additional information in the report. function enrich-report() { build_datetime=${BUILD_DATETIME:-$(date -u +'%Y-%m-%dT%H:%M:%S%z')} diff --git a/scripts/reports/perform-static-analysis.sh b/scripts/reports/perform-static-analysis.sh index f52c068b..f4f6fb6b 100755 --- a/scripts/reports/perform-static-analysis.sh +++ b/scripts/reports/perform-static-analysis.sh @@ -17,8 +17,8 @@ set -euo pipefail # SONAR_TOKEN=token # SonarCloud token # # Options: -# VERBOSE=true # Show all the executed commands, default is 'false' # FORCE_USE_DOCKER=true # If set to true the command is run in a Docker container, default is 'false' +# VERBOSE=true # Show all the executed commands, default is 'false' # ============================================================================== diff --git a/scripts/reports/scan-vulnerabilities.sh b/scripts/reports/scan-vulnerabilities.sh index 22a8cfd8..e2f212bf 100755 --- a/scripts/reports/scan-vulnerabilities.sh +++ b/scripts/reports/scan-vulnerabilities.sh @@ -5,18 +5,19 @@ set -euo pipefail # Script to scan an SBOM file for CVEs (Common Vulnerabilities and Exposures). +# This is a grype command wrapper. It will run grype natively if it is +# installed, otherwise it will run it in a Docker container. # # Usage: # $ ./scan-vulnerabilities.sh # # Options: -# VERBOSE=true # Show all the executed commands, default is `false` # BUILD_DATETIME=%Y-%m-%dT%H:%M:%S%z # Build datetime, default is `date -u +'%Y-%m-%dT%H:%M:%S%z'` - -# ============================================================================== - -# SEE: https://github.com/anchore/grype/pkgs/container/grype, use the `linux/amd64` os/arch -image_version=v0.69.1@sha256:d41fcb371d0af59f311e72123dff46900ebd6d0482391b5a830853ee4f9d1a76 +# FORCE_USE_DOCKER=true # If set to true the command is run in a Docker container, default is 'false' +# VERBOSE=true # Show all the executed commands, default is `false` +# +# Depends on: +# $ ./generate-sbom.sh # ============================================================================== @@ -28,18 +29,45 @@ function main() { enrich-report } +# Create the report. function create-report() { + if command -v grype > /dev/null 2>&1 && ! is-arg-true "${FORCE_USE_DOCKER:-false}"; then + cli-run-grype + else + docker-run-grype + fi +} + +# Run grype natively. +function cli-run-grype() { + + grype \ + sbom:"$PWD/sbom-repository-report.json" \ + --config "$PWD/scripts/config/grype.yaml" \ + --output json \ + --file "$PWD/vulnerabilities-repository-report.tmp.json" +} + +# Run grype in a Docker container. +function docker-run-grype() { + + # shellcheck disable=SC1091 + source ./scripts/docker/docker.lib.sh + + # shellcheck disable=SC2155 + local image=$(name=ghcr.io/anchore/grype docker-get-image-version-and-pull) docker run --rm --platform linux/amd64 \ - --volume "$PWD":/scan \ + --volume "$PWD":/workdir \ --volume /tmp/grype/db:/.cache/grype/db \ - ghcr.io/anchore/grype:$image_version \ - sbom:/scan/sbom-repository-report.json \ - --config /scan/scripts/config/grype.yaml \ + "$image" \ + sbom:/workdir/sbom-repository-report.json \ + --config /workdir/scripts/config/grype.yaml \ --output json \ - --file /scan/vulnerabilities-repository-report.tmp.json + --file /workdir/vulnerabilities-repository-report.tmp.json } +# Include additional information in the report. function enrich-report() { build_datetime=${BUILD_DATETIME:-$(date -u +'%Y-%m-%dT%H:%M:%S%z')} diff --git a/scripts/shellscript-linter.sh b/scripts/shellscript-linter.sh index 15443fe8..66bb4513 100755 --- a/scripts/shellscript-linter.sh +++ b/scripts/shellscript-linter.sh @@ -12,8 +12,8 @@ set -euo pipefail # # Arguments (provided as environment variables): # file=shellscript # Path to the shell script to lint, relative to the project's top-level directory, default is itself -# VERBOSE=true # Show all the executed commands, default is 'false' # FORCE_USE_DOCKER=true # If set to true the command is run in a Docker container, default is 'false' +# VERBOSE=true # Show all the executed commands, default is 'false' # ============================================================================== diff --git a/scripts/terraform/terraform.lib.sh b/scripts/terraform/terraform.lib.sh index b19aa28a..7793b9b0 100644 --- a/scripts/terraform/terraform.lib.sh +++ b/scripts/terraform/terraform.lib.sh @@ -15,7 +15,7 @@ set -euo pipefail # Initialise Terraform. # Arguments (provided as environment variables): # dir=[path to a directory where the command will be executed, relative to the project's top-level directory, default is '.'] -#  opts=[options to pass to the Terraform init command, default is none/empty] +# opts=[options to pass to the Terraform init command, default is none/empty] function terraform-init() { _terraform init # 'dir' and 'opts' are passed to the function as environment variables, if set @@ -24,7 +24,7 @@ function terraform-init() { # Plan Terraform changes. # Arguments (provided as environment variables): # dir=[path to a directory where the command will be executed, relative to the project's top-level directory, default is '.'] -#  opts=[options to pass to the Terraform plan command, default is none/empty] +# opts=[options to pass to the Terraform plan command, default is none/empty] function terraform-plan() { _terraform plan # 'dir' and 'opts' are passed to the function as environment variables, if set @@ -33,7 +33,7 @@ function terraform-plan() { # Apply Terraform changes. # Arguments (provided as environment variables): # dir=[path to a directory where the command will be executed, relative to the project's top-level directory, default is '.'] -#  opts=[options to pass to the Terraform apply command, default is none/empty] +# opts=[options to pass to the Terraform apply command, default is none/empty] function terraform-apply() { _terraform apply # 'dir' and 'opts' are passed to the function as environment variables, if set @@ -42,7 +42,7 @@ function terraform-apply() { # Destroy Terraform resources. # Arguments (provided as environment variables): # dir=[path to a directory where the command will be executed, relative to the project's top-level directory, default is '.'] -#  opts=[options to pass to the Terraform destroy command, default is none/empty] +# opts=[options to pass to the Terraform destroy command, default is none/empty] function terraform-destroy() { _terraform apply -destroy # 'dir' and 'opts' are passed to the function as environment variables, if set @@ -51,7 +51,7 @@ function terraform-destroy() { # Format Terraform code. # Arguments (provided as environment variables): # dir=[path to a directory where the command will be executed, relative to the project's top-level directory, default is '.'] -#  opts=[options to pass to the Terraform fmt command, default is '-recursive'] +# opts=[options to pass to the Terraform fmt command, default is '-recursive'] function terraform-fmt() { _terraform fmt -recursive # 'dir' and 'opts' are passed to the function as environment variables, if set @@ -60,7 +60,7 @@ function terraform-fmt() { # Validate Terraform code. # Arguments (provided as environment variables): # dir=[path to a directory where the command will be executed, relative to the project's top-level directory, default is '.'] -#  opts=[options to pass to the Terraform validate command, default is none/empty] +# opts=[options to pass to the Terraform validate command, default is none/empty] function terraform-validate() { _terraform validate # 'dir' and 'opts' are passed to the function as environment variables, if set diff --git a/scripts/terraform/terraform.mk b/scripts/terraform/terraform.mk index 2042e7c0..120a0591 100644 --- a/scripts/terraform/terraform.mk +++ b/scripts/terraform/terraform.mk @@ -4,7 +4,7 @@ # Custom implementation - implementation of a make target should not exceed 5 lines of effective code. # In most cases there should be no need to modify the existing make targets. -terraform-init: # Initialise Terraform - optional: terraform_dir|dir=[path to a directory where the command will be executed, relative to the project's top-level directory, default is one of the module variables or the example directory, if not set], terraform_opts|opts=[options to pass to the Terraform init command, default is none/empty] @Development +terraform-init: # Initialise Terraform - optional: terraform_dir|dir=[path to a directory where the command will be executed, relative to the project's top-level directory, default is one of the module variables or the example directory, if not set], terraform_opts|opts=[options to pass to the Terraform init command, default is none/empty] @Development make _terraform cmd="init" \ dir=$(or ${terraform_dir}, ${dir}) \ opts=$(or ${terraform_opts}, ${opts}) @@ -40,7 +40,7 @@ clean:: # Remove Terraform files (terraform) - optional: terraform_dir|dir=[path dir=$(or ${terraform_dir}, ${dir}) \ opts=$(or ${terraform_opts}, ${opts}) -_terraform: # Terraform command wrapper - mandatory: cmd=[command to execute]; optional: dir=[path to a directory where the command will be executed, relative to the project's top-level directory, default is one of the module variables or the example directory, if not set], opts=[options to pass to the Terraform command, default is none/empty] +_terraform: # Terraform command wrapper - mandatory: cmd=[command to execute]; optional: dir=[path to a directory where the command will be executed, relative to the project's top-level directory, default is one of the module variables or the example directory, if not set], opts=[options to pass to the Terraform command, default is none/empty] # 'TERRAFORM_STACK' is passed to the functions as environment variable TERRAFORM_STACK=$(or ${TERRAFORM_STACK}, $(or ${terraform_stack}, $(or ${STACK}, $(or ${stack}, scripts/terraform/examples/terraform-state-aws-s3)))) dir=$(or ${dir}, ${TERRAFORM_STACK}) diff --git a/scripts/terraform/terraform.sh b/scripts/terraform/terraform.sh index 9fdff54f..e8c0ec90 100755 --- a/scripts/terraform/terraform.sh +++ b/scripts/terraform/terraform.sh @@ -12,8 +12,8 @@ set -euo pipefail # # Options: # cmd=command # Terraform command to execute -# VERBOSE=true # Show all the executed commands, default is 'false' # FORCE_USE_DOCKER=true # If set to true the command is run in a Docker container, default is 'false' +# VERBOSE=true # Show all the executed commands, default is 'false' # ============================================================================== diff --git a/scripts/tests/unit.sh b/scripts/tests/unit.sh index ef2989ce..c589be5b 100755 --- a/scripts/tests/unit.sh +++ b/scripts/tests/unit.sh @@ -15,6 +15,6 @@ cd "$(git rev-parse --show-toplevel)" # # or whatever is appropriate to your project. You should *only* run your fast # tests from here. If you want to run other test suites, see the predefined -# tasks in scripts/test.mk. +# tasks in scripts/test.mk. echo "Unit tests are not yet implemented. See scripts/tests/unit.sh for more." From 24e74deaffd78246f62f122acec4f88798ac7a5c Mon Sep 17 00:00:00 2001 From: Dan Stefaniuk Date: Tue, 17 Oct 2023 11:22:28 +0100 Subject: [PATCH 2/5] Address peer review feedback --- scripts/githooks/check-file-format.sh | 13 ++++++------- scripts/githooks/check-markdown-format.sh | 9 +++++---- scripts/githooks/check-terraform-format.sh | 2 +- scripts/githooks/scan-secrets.sh | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/scripts/githooks/check-file-format.sh b/scripts/githooks/check-file-format.sh index 75029641..83d0ceb9 100755 --- a/scripts/githooks/check-file-format.sh +++ b/scripts/githooks/check-file-format.sh @@ -11,12 +11,14 @@ set -euo pipefail # installed, otherwise it will run it in a Docker container. # # Usage: -# $ check={all,staged-changes,working-tree-changes,branch} [dry_run=true] ./check-file-format.sh +# $ [options] ./check-file-format.sh # # Options: -# BRANCH_NAME=other-branch-than-main # Branch to compare with, default is `origin/main` -# FORCE_USE_DOCKER=true # If set to true the command is run in a Docker container, default is 'false' -# VERBOSE=true # Show all the executed commands, default is `false` +# check={all,staged-changes,working-tree-changes,branch} # Check mode, default is 'working-tree-changes' +# dry_run=true # Do not check, run dry run only, default is 'false' +# BRANCH_NAME=other-branch-than-main # Branch to compare with, default is `origin/main` +# FORCE_USE_DOCKER=true # If set to true the command is run in a Docker container, default is 'false' +# VERBOSE=true # Show all the executed commands, default is `false` # # Exit codes: # 0 - All files are formatted correctly @@ -31,9 +33,6 @@ set -euo pipefail # check=working-tree-changes: check modified, unstaged files. This is the default. # check=branch: check for all changes since branching from $BRANCH_NAME # -# If the `dry_run` parameter is set to a truthy value, the list of -# files that ec would check is output, with no check done. -# # Notes: # Please make sure to enable EditorConfig linting in your IDE. For the # Visual Studio Code editor it is `editorconfig.editorconfig` that is already diff --git a/scripts/githooks/check-markdown-format.sh b/scripts/githooks/check-markdown-format.sh index 311e3e3a..832e990c 100755 --- a/scripts/githooks/check-markdown-format.sh +++ b/scripts/githooks/check-markdown-format.sh @@ -10,12 +10,13 @@ set -euo pipefail # container. # # Usage: -# $ check={all,staged-changes,working-tree-changes,branch} ./check-markdown-format.sh +# $ [options] ./check-markdown-format.sh # # Options: -# BRANCH_NAME=other-branch-than-main # Branch to compare with, default is `origin/main` -# FORCE_USE_DOCKER=true # If set to true the command is run in a Docker container, default is 'false' -# VERBOSE=true # Show all the executed commands, default is `false` +# check={all,staged-changes,working-tree-changes,branch} # Check mode, default is 'working-tree-changes' +# BRANCH_NAME=other-branch-than-main # Branch to compare with, default is `origin/main` +# FORCE_USE_DOCKER=true # If set to true the command is run in a Docker container, default is 'false' +# VERBOSE=true # Show all the executed commands, default is `false` # # Exit codes: # 0 - All files are formatted correctly diff --git a/scripts/githooks/check-terraform-format.sh b/scripts/githooks/check-terraform-format.sh index ec40f86d..7255e512 100755 --- a/scripts/githooks/check-terraform-format.sh +++ b/scripts/githooks/check-terraform-format.sh @@ -7,7 +7,7 @@ set -euo pipefail # Pre-commit git hook to check format Terraform code. # # Usage: -# $ ./check-terraform-format.sh +# $ [options] ./check-terraform-format.sh # # Options: # check_only=true # Do not format, run check only, default is 'false' diff --git a/scripts/githooks/scan-secrets.sh b/scripts/githooks/scan-secrets.sh index e076fb87..ee9637f5 100755 --- a/scripts/githooks/scan-secrets.sh +++ b/scripts/githooks/scan-secrets.sh @@ -9,7 +9,7 @@ set -euo pipefail # otherwise it will run it in a Docker container. # # Usage: -# $ ./scan-secrets.sh +# $ [options] ./scan-secrets.sh # # Options: # check={whole-history,last-commit,staged-changes} # Type of the check to run, default is 'staged-changes' From 3561dcb106adebc02f92037ef51fd81ce4bf5b3b Mon Sep 17 00:00:00 2001 From: Dan Stefaniuk Date: Tue, 17 Oct 2023 11:44:30 +0100 Subject: [PATCH 3/5] Address peer review feedback --- scripts/docker/dockerfile-linter.sh | 10 +++++----- scripts/githooks/check-file-format.sh | 8 ++++---- scripts/githooks/check-markdown-format.sh | 8 ++++---- scripts/githooks/scan-secrets.sh | 8 ++++---- scripts/reports/cloc-repository.sh | 14 +++++--------- scripts/reports/generate-sbom.sh | 14 +++++--------- scripts/reports/perform-static-analysis.sh | 12 +++++------- scripts/reports/scan-vulnerabilities.sh | 14 +++++--------- scripts/shellscript-linter.sh | 10 +++++----- scripts/terraform/terraform.sh | 10 +++++----- 10 files changed, 47 insertions(+), 61 deletions(-) diff --git a/scripts/docker/dockerfile-linter.sh b/scripts/docker/dockerfile-linter.sh index 9e74a8fb..7e8c75f0 100755 --- a/scripts/docker/dockerfile-linter.sh +++ b/scripts/docker/dockerfile-linter.sh @@ -8,7 +8,7 @@ set -euo pipefail # otherwise it will run it in a Docker container. # # Usage: -# $ ./dockerfile-linter.sh +# $ [options] ./dockerfile-linter.sh # # Arguments (provided as environment variables): # file=Dockerfile # Path to the Dockerfile to lint, relative to the project's top-level directory, default is './Dockerfile.effective' @@ -23,16 +23,16 @@ function main() { local file=${file:-./Dockerfile.effective} if command -v hadolint > /dev/null 2>&1 && ! is-arg-true "${FORCE_USE_DOCKER:-false}"; then - file="$file" cli-run-hadolint + file="$file" run-hadolint-natively else - file="$file" docker-run-hadolint + file="$file" run-hadolint-in-docker fi } # Run hadolint natively. # Arguments (provided as environment variables): # file=[path to the Dockerfile to lint, relative to the project's top-level directory] -function cli-run-hadolint() { +function run-hadolint-natively() { # shellcheck disable=SC2001 hadolint "$(echo "$file" | sed "s#$PWD#.#")" @@ -41,7 +41,7 @@ function cli-run-hadolint() { # Run hadolint in a Docker container. # Arguments (provided as environment variables): # file=[path to the Dockerfile to lint, relative to the project's top-level directory] -function docker-run-hadolint() { +function run-hadolint-in-docker() { # shellcheck disable=SC1091 source ./scripts/docker/docker.lib.sh diff --git a/scripts/githooks/check-file-format.sh b/scripts/githooks/check-file-format.sh index 83d0ceb9..fd466adb 100755 --- a/scripts/githooks/check-file-format.sh +++ b/scripts/githooks/check-file-format.sh @@ -67,9 +67,9 @@ function main() { esac if command -v editorconfig > /dev/null 2>&1 && ! is-arg-true "${FORCE_USE_DOCKER:-false}"; then - filter="$filter" dry_run_opt="${dry_run_opt:-}" cli-run-editorconfig + filter="$filter" dry_run_opt="${dry_run_opt:-}" run-editorconfig-natively else - filter="$filter" dry_run_opt="${dry_run_opt:-}" docker-run-editorconfig + filter="$filter" dry_run_opt="${dry_run_opt:-}" run-editorconfig-in-docker fi } @@ -77,7 +77,7 @@ function main() { # Arguments (provided as environment variables): # dry_run_opt=[dry run option] # filter=[filter for files to check] -function cli-run-editorconfig() { +function run-editorconfig-natively() { # shellcheck disable=SC2046,SC2086 editorconfig \ @@ -88,7 +88,7 @@ function cli-run-editorconfig() { # Arguments (provided as environment variables): # dry_run_opt=[dry run option] # filter=[filter for files to check] -function docker-run-editorconfig() { +function run-editorconfig-in-docker() { # shellcheck disable=SC1091 source ./scripts/docker/docker.lib.sh diff --git a/scripts/githooks/check-markdown-format.sh b/scripts/githooks/check-markdown-format.sh index 832e990c..698df4a5 100755 --- a/scripts/githooks/check-markdown-format.sh +++ b/scripts/githooks/check-markdown-format.sh @@ -53,9 +53,9 @@ function main() { if [ -n "$files" ]; then if command -v markdownlint > /dev/null 2>&1 && ! is-arg-true "${FORCE_USE_DOCKER:-false}"; then - files="$files" cli-run-markdownlint + files="$files" run-markdownlint-natively else - files="$files" docker-run-markdownlint + files="$files" run-markdownlint-in-docker fi fi } @@ -63,7 +63,7 @@ function main() { # Run markdownlint natively. # Arguments (provided as environment variables): # files=[files to check] -function cli-run-markdownlint() { +function run-markdownlint-natively() { # shellcheck disable=SC2086 markdownlint \ @@ -74,7 +74,7 @@ function cli-run-markdownlint() { # Run markdownlint in a Docker container. # Arguments (provided as environment variables): # files=[files to check] -function docker-run-markdownlint() { +function run-markdownlint-in-docker() { # shellcheck disable=SC1091 source ./scripts/docker/docker.lib.sh diff --git a/scripts/githooks/scan-secrets.sh b/scripts/githooks/scan-secrets.sh index ee9637f5..06155b8a 100755 --- a/scripts/githooks/scan-secrets.sh +++ b/scripts/githooks/scan-secrets.sh @@ -29,10 +29,10 @@ function main() { if command -v gitleaks > /dev/null 2>&1 && ! is-arg-true "${FORCE_USE_DOCKER:-false}"; then dir="$PWD" - cmd="$(get-cmd-to-run)" cli-run-gitleaks + cmd="$(get-cmd-to-run)" run-gitleaks-natively else dir="/workdir" - cmd="$(get-cmd-to-run)" docker-run-gitleaks + cmd="$(get-cmd-to-run)" run-gitleaks-in-docker fi } @@ -66,7 +66,7 @@ function get-cmd-to-run() { # Run Gitleaks natively. # Arguments (provided as environment variables): # cmd=[command to run] -function cli-run-gitleaks() { +function run-gitleaks-natively() { # shellcheck disable=SC2086 gitleaks $cmd @@ -76,7 +76,7 @@ function cli-run-gitleaks() { # Arguments (provided as environment variables): # cmd=[command to run] # dir=[directory to mount as a volume] -function docker-run-gitleaks() { +function run-gitleaks-in-docker() { # shellcheck disable=SC1091 source ./scripts/docker/docker.lib.sh diff --git a/scripts/reports/cloc-repository.sh b/scripts/reports/cloc-repository.sh index c510d98f..1c80b4a7 100755 --- a/scripts/reports/cloc-repository.sh +++ b/scripts/reports/cloc-repository.sh @@ -9,7 +9,7 @@ set -euo pipefail # Docker container. # # Usage: -# $ ./cloc-repository.sh +# $ [options] ./cloc-repository.sh # # Options: # BUILD_DATETIME=%Y-%m-%dT%H:%M:%S%z # Build datetime, default is `date -u +'%Y-%m-%dT%H:%M:%S%z'` @@ -26,13 +26,12 @@ function main() { enrich-report } -# Create the report. function create-report() { if command -v gocloc > /dev/null 2>&1 && ! is-arg-true "${FORCE_USE_DOCKER:-false}"; then - cli-run-gocloc + run-gocloc-natively else - docker-run-gocloc + run-gocloc-in-docker fi # shellcheck disable=SC2002 cat cloc-report.tmp.json \ @@ -41,14 +40,12 @@ function create-report() { | column -t } -# Run gocloc natively. -function cli-run-gocloc() { +function run-gocloc-natively() { gocloc --output-type=json . > cloc-report.tmp.json } -# Run gocloc in a Docker container. -function docker-run-gocloc() { +function run-gocloc-in-docker() { # shellcheck disable=SC1091 source ./scripts/docker/docker.lib.sh @@ -63,7 +60,6 @@ function docker-run-gocloc() { > cloc-report.tmp.json } -# Include additional information in the report. function enrich-report() { build_datetime=${BUILD_DATETIME:-$(date -u +'%Y-%m-%dT%H:%M:%S%z')} diff --git a/scripts/reports/generate-sbom.sh b/scripts/reports/generate-sbom.sh index ed861d3b..117c1cf0 100755 --- a/scripts/reports/generate-sbom.sh +++ b/scripts/reports/generate-sbom.sh @@ -10,7 +10,7 @@ set -euo pipefail # it in a Docker container. # # Usage: -# $ ./generate-sbom.sh +# $ [options] ./generate-sbom.sh # # Options: # BUILD_DATETIME=%Y-%m-%dT%H:%M:%S%z # Build datetime, default is `date -u +'%Y-%m-%dT%H:%M:%S%z'` @@ -27,26 +27,23 @@ function main() { enrich-report } -# Create the report. function create-report() { if command -v syft > /dev/null 2>&1 && ! is-arg-true "${FORCE_USE_DOCKER:-false}"; then - cli-run-syft + run-syft-natively else - docker-run-syft + run-syft-in-docker fi } -# Run syft natively. -function cli-run-syft() { +function run-syft-natively() { syft packages dir:"$PWD" \ --config "$PWD/scripts/config/syft.yaml" \ --output spdx-json="$PWD/sbom-repository-report.tmp.json" } -# Run syft in a Docker container. -function docker-run-syft() { +function run-syft-in-docker() { # shellcheck disable=SC1091 source ./scripts/docker/docker.lib.sh @@ -61,7 +58,6 @@ function docker-run-syft() { --output spdx-json=/workdir/sbom-repository-report.tmp.json } -# Include additional information in the report. function enrich-report() { build_datetime=${BUILD_DATETIME:-$(date -u +'%Y-%m-%dT%H:%M:%S%z')} diff --git a/scripts/reports/perform-static-analysis.sh b/scripts/reports/perform-static-analysis.sh index f4f6fb6b..2426e6d0 100755 --- a/scripts/reports/perform-static-analysis.sh +++ b/scripts/reports/perform-static-analysis.sh @@ -8,7 +8,7 @@ set -euo pipefail # report to SonarCloud. # # Usage: -# $ ./perform-static-analysis.sh +# $ [options] ./perform-static-analysis.sh # # Expects: # BRANCH_NAME=branch-name # Branch to report on @@ -27,14 +27,13 @@ function main() { cd "$(git rev-parse --show-toplevel)" if command -v sonar-scanner > /dev/null 2>&1 && ! is-arg-true "${FORCE_USE_DOCKER:-false}"; then - cli-run-sonar-scanner + run-sonar-scanner-natively else - docker-run-sonar-scanner + run-sonar-scanner-in-docker fi } -# Run Sonar Scanner natively. -function cli-run-sonar-scanner() { +function run-sonar-scanner-natively() { sonar-scanner \ -Dproject.settings="$PWD/scripts/config/sonar-scanner.properties" \ @@ -44,8 +43,7 @@ function cli-run-sonar-scanner() { -Dsonar.token="$SONAR_TOKEN" } -# Run Sonar Scanner in a Docker container. -function docker-run-sonar-scanner() { +function run-sonar-scanner-in-docker() { # shellcheck disable=SC1091 source ./scripts/docker/docker.lib.sh diff --git a/scripts/reports/scan-vulnerabilities.sh b/scripts/reports/scan-vulnerabilities.sh index e2f212bf..ef4e5020 100755 --- a/scripts/reports/scan-vulnerabilities.sh +++ b/scripts/reports/scan-vulnerabilities.sh @@ -9,7 +9,7 @@ set -euo pipefail # installed, otherwise it will run it in a Docker container. # # Usage: -# $ ./scan-vulnerabilities.sh +# $ [options] ./scan-vulnerabilities.sh # # Options: # BUILD_DATETIME=%Y-%m-%dT%H:%M:%S%z # Build datetime, default is `date -u +'%Y-%m-%dT%H:%M:%S%z'` @@ -29,18 +29,16 @@ function main() { enrich-report } -# Create the report. function create-report() { if command -v grype > /dev/null 2>&1 && ! is-arg-true "${FORCE_USE_DOCKER:-false}"; then - cli-run-grype + run-grype-natively else - docker-run-grype + run-grype-in-docker fi } -# Run grype natively. -function cli-run-grype() { +function run-grype-natively() { grype \ sbom:"$PWD/sbom-repository-report.json" \ @@ -49,8 +47,7 @@ function cli-run-grype() { --file "$PWD/vulnerabilities-repository-report.tmp.json" } -# Run grype in a Docker container. -function docker-run-grype() { +function run-grype-in-docker() { # shellcheck disable=SC1091 source ./scripts/docker/docker.lib.sh @@ -67,7 +64,6 @@ function docker-run-grype() { --file /workdir/vulnerabilities-repository-report.tmp.json } -# Include additional information in the report. function enrich-report() { build_datetime=${BUILD_DATETIME:-$(date -u +'%Y-%m-%dT%H:%M:%S%z')} diff --git a/scripts/shellscript-linter.sh b/scripts/shellscript-linter.sh index 66bb4513..8b3fe09c 100755 --- a/scripts/shellscript-linter.sh +++ b/scripts/shellscript-linter.sh @@ -8,7 +8,7 @@ set -euo pipefail # installed, otherwise it will run it in a Docker container. # # Usage: -# $ ./shellscript-linter.sh +# $ [options] ./shellscript-linter.sh # # Arguments (provided as environment variables): # file=shellscript # Path to the shell script to lint, relative to the project's top-level directory, default is itself @@ -24,16 +24,16 @@ function main() { [ -z "${file:-}" ] && echo "WARNING: 'file' variable not set, defaulting to itself" local file=${file:-scripts/shellscript-linter.sh} if command -v shellcheck > /dev/null 2>&1 && ! is-arg-true "${FORCE_USE_DOCKER:-false}"; then - file="$file" cli-run-shellcheck + file="$file" run-shellcheck-natively else - file="$file" docker-run-shellcheck + file="$file" run-shellcheck-in-docker fi } # Run ShellCheck natively. # Arguments (provided as environment variables): # file=[path to the shell script to lint, relative to the project's top-level directory] -function cli-run-shellcheck() { +function run-shellcheck-natively() { # shellcheck disable=SC2001 shellcheck "$(echo "$file" | sed "s#$PWD#.#")" @@ -42,7 +42,7 @@ function cli-run-shellcheck() { # Run ShellCheck in a Docker container. # Arguments (provided as environment variables): # file=[path to the shell script to lint, relative to the project's top-level directory] -function docker-run-shellcheck() { +function run-shellcheck-in-docker() { # shellcheck disable=SC1091 source ./scripts/docker/docker.lib.sh diff --git a/scripts/terraform/terraform.sh b/scripts/terraform/terraform.sh index e8c0ec90..73f37c1a 100755 --- a/scripts/terraform/terraform.sh +++ b/scripts/terraform/terraform.sh @@ -8,7 +8,7 @@ set -euo pipefail # installed, otherwise it will run it in a Docker container. # # Usage: -# $ ./terraform.sh +# $ [options] ./terraform.sh # # Options: # cmd=command # Terraform command to execute @@ -23,16 +23,16 @@ function main() { if command -v terraform > /dev/null 2>&1 && ! is-arg-true "${FORCE_USE_DOCKER:-false}"; then # shellcheck disable=SC2154 - cmd=$cmd cli-run-terraform + cmd=$cmd run-terraform-natively else - cmd=$cmd docker-run-terraform + cmd=$cmd run-terraform-in-docker fi } # Run Terraform natively. # Arguments (provided as environment variables): # cmd=[Terraform command to execute] -function cli-run-terraform() { +function run-terraform-natively() { # shellcheck disable=SC2086 terraform $cmd @@ -41,7 +41,7 @@ function cli-run-terraform() { # Run Terraform in a Docker container. # Arguments (provided as environment variables): # cmd=[Terraform command to execute] -function docker-run-terraform() { +function run-terraform-in-docker() { # shellcheck disable=SC1091 source ./scripts/docker/docker.lib.sh From 77f5881ef128aa3684517a7ea23218a0758fc8cf Mon Sep 17 00:00:00 2001 From: Dan Stefaniuk Date: Tue, 17 Oct 2023 11:57:23 +0100 Subject: [PATCH 4/5] Address peer review feedback --- .../action.yaml | 12 ++++++------ .github/actions/scan-dependencies/action.yaml | 2 +- .github/workflows/stage-1-commit.yaml | 4 ++-- docs/user-guides/Scan_dependencies.md | 4 ++-- docs/user-guides/Test_GitHub_Actions_locally.md | 4 ++-- scripts/init.mk | 4 +--- ...epository.sh => create-lines-of-code-report.sh} | 14 +++++++------- .../{generate-sbom.sh => create-sbom-report.sh} | 2 +- scripts/reports/scan-vulnerabilities.sh | 2 +- 9 files changed, 23 insertions(+), 25 deletions(-) rename .github/actions/{cloc-repository => create-lines-of-code-report}/action.yaml (85%) rename scripts/reports/{cloc-repository.sh => create-lines-of-code-report.sh} (90%) rename scripts/reports/{generate-sbom.sh => create-sbom-report.sh} (98%) diff --git a/.github/actions/cloc-repository/action.yaml b/.github/actions/create-lines-of-code-report/action.yaml similarity index 85% rename from .github/actions/cloc-repository/action.yaml rename to .github/actions/create-lines-of-code-report/action.yaml index 03c55831..452b432b 100644 --- a/.github/actions/cloc-repository/action.yaml +++ b/.github/actions/create-lines-of-code-report/action.yaml @@ -26,16 +26,16 @@ runs: shell: bash run: | export BUILD_DATETIME=${{ inputs.build_datetime }} - ./scripts/reports/cloc-repository.sh + ./scripts/reports/create-lines-of-code-report.sh - name: "Compress CLOC report" shell: bash - run: zip cloc-report.json.zip cloc-report.json + run: zip lines-of-code-report.json.zip lines-of-code-report.json - name: "Upload CLOC report as an artefact" if: ${{ !env.ACT }} uses: actions/upload-artifact@v3 with: - name: cloc-report.json.zip - path: ./cloc-report.json.zip + name: lines-of-code-report.json.zip + path: ./lines-of-code-report.json.zip retention-days: 21 - name: "Check prerequisites for sending the report" shell: bash @@ -53,5 +53,5 @@ runs: if: steps.check.outputs.secrets_exist == 'true' run: | aws s3 cp \ - ./cloc-report.json.zip \ - ${{ inputs.idp_aws_report_upload_bucket_endpoint }}/${{ inputs.build_timestamp }}-cloc-report.json.zip + ./lines-of-code-report.json.zip \ + ${{ inputs.idp_aws_report_upload_bucket_endpoint }}/${{ inputs.build_timestamp }}-lines-of-code-report.json.zip diff --git a/.github/actions/scan-dependencies/action.yaml b/.github/actions/scan-dependencies/action.yaml index bac9cdbc..925ba096 100644 --- a/.github/actions/scan-dependencies/action.yaml +++ b/.github/actions/scan-dependencies/action.yaml @@ -26,7 +26,7 @@ runs: shell: bash run: | export BUILD_DATETIME=${{ inputs.build_datetime }} - ./scripts/reports/generate-sbom.sh + ./scripts/reports/create-sbom-report.sh - name: "Compress SBOM report" shell: bash run: zip sbom-repository-report.json.zip sbom-repository-report.json diff --git a/.github/workflows/stage-1-commit.yaml b/.github/workflows/stage-1-commit.yaml index 81fdb2cb..131cc1c0 100644 --- a/.github/workflows/stage-1-commit.yaml +++ b/.github/workflows/stage-1-commit.yaml @@ -75,7 +75,7 @@ jobs: uses: actions/checkout@v4 - name: "Lint Terraform" uses: ./.github/actions/lint-terraform - cloc-repository: + count-lines-of-code: name: "Count lines of code" runs-on: ubuntu-latest permissions: @@ -86,7 +86,7 @@ jobs: - name: "Checkout code" uses: actions/checkout@v4 - name: "Count lines of code" - uses: ./.github/actions/cloc-repository + uses: ./.github/actions/create-lines-of-code-report with: build_datetime: "${{ inputs.build_datetime }}" build_timestamp: "${{ inputs.build_timestamp }}" diff --git a/docs/user-guides/Scan_dependencies.md b/docs/user-guides/Scan_dependencies.md index 8872aeca..25aff559 100644 --- a/docs/user-guides/Scan_dependencies.md +++ b/docs/user-guides/Scan_dependencies.md @@ -15,7 +15,7 @@ In modern software development, leveraging third-party dependencies is a common ## Key files -- [generate-sbom.sh](../../scripts/reports/generate-sbom.sh): A shell script that generates SBOM (Software Bill of Materials) +- [create-sbom-report.sh](../../scripts/reports/create-sbom-report.sh): A shell script that generates SBOM (Software Bill of Materials) - [syft.yaml](../../scripts/config/syft.yaml): A configuration file for the SBOM generator - [scan-vulnerabilities.sh](../../scripts/reports/scan-vulnerabilities.sh): A shell script that performs CVE analysis - [grype.yaml](../../scripts/config/grype.yaml): A configuration file for the CVE scanner @@ -41,7 +41,7 @@ You can run and test the process locally on a developer's workstation using the SBOM generator ```shell -./scripts/reports/generate-sbom.sh +./scripts/reports/create-sbom-report.sh cat sbom-repository-report.json | jq ``` diff --git a/docs/user-guides/Test_GitHub_Actions_locally.md b/docs/user-guides/Test_GitHub_Actions_locally.md index 0f35f1d9..2da643b2 100644 --- a/docs/user-guides/Test_GitHub_Actions_locally.md +++ b/docs/user-guides/Test_GitHub_Actions_locally.md @@ -28,7 +28,7 @@ The following command-line tools are expected to be installed: Here is an example on how to run a GitHub workflow job: ```shell -$ make runner-act workflow="stage-1-commit" job="cloc-repository" +$ make runner-act workflow="stage-1-commit" job="create-lines-of-code-report" [Commit stage/Count lines of code] 🚀 Start image=ghcr.io/nhs-england-tools/github-runner-image:20230101-abcdef0-rt [Commit stage/Count lines of code] 🐳 docker pull image=ghcr.io/nhs-england-tools/github-runner-image:20230101-abcdef0-rt platform=linux/amd64 username= forcePull=false @@ -42,7 +42,7 @@ $ make runner-act workflow="stage-1-commit" job="cloc-repository" [Commit stage/Count lines of code] ✅ Success - Main Create CLOC report [Commit stage/Count lines of code] ⭐ Run Main Compress CLOC report [Commit stage/Count lines of code] 🐳 docker exec cmd=[bash --noprofile --norc -e -o pipefail /var/run/act/workflow/1-composite-1.sh] user= workdir= -| updating: cloc-report.json (deflated 68%) +| updating: lines-of-code-report.json (deflated 68%) [Commit stage/Count lines of code] ✅ Success - Main Compress CLOC report [Commit stage/Count lines of code] ☁ git clone 'https://github.com/actions/upload-artifact' # ref=v3 [Commit stage/Count lines of code] ⭐ Run Main Check prerequisites for sending the report diff --git a/scripts/init.mk b/scripts/init.mk index eca74413..5800e1e8 100644 --- a/scripts/init.mk +++ b/scripts/init.mk @@ -47,9 +47,7 @@ _install-dependency: # Install asdf dependency - mandatory: name=[listed in the clean:: # Remove all generated and temporary files (common) @Operations rm -rf \ .scannerwork \ - *cloc-report*.json \ - *sbom*report*.json \ - *vulnerabilities*report*.json \ + *report*.json \ *report*json.zip \ docs/diagrams/.*.bkp \ docs/diagrams/.*.dtmp \ diff --git a/scripts/reports/cloc-repository.sh b/scripts/reports/create-lines-of-code-report.sh similarity index 90% rename from scripts/reports/cloc-repository.sh rename to scripts/reports/create-lines-of-code-report.sh index 1c80b4a7..01645c7d 100755 --- a/scripts/reports/cloc-repository.sh +++ b/scripts/reports/create-lines-of-code-report.sh @@ -9,7 +9,7 @@ set -euo pipefail # Docker container. # # Usage: -# $ [options] ./cloc-repository.sh +# $ [options] ./create-lines-of-code-report.sh # # Options: # BUILD_DATETIME=%Y-%m-%dT%H:%M:%S%z # Build datetime, default is `date -u +'%Y-%m-%dT%H:%M:%S%z'` @@ -34,7 +34,7 @@ function create-report() { run-gocloc-in-docker fi # shellcheck disable=SC2002 - cat cloc-report.tmp.json \ + cat lines-of-code-report.tmp.json \ | jq -r '["Language","files","blank","comment","code"],["--------"],(.languages[]|[.name,.files,.blank,.comment,.code]),["-----"],(.total|["TOTAL",.files,.blank,.comment,.code])|@tsv' \ | sed 's/Plain Text/Plaintext/g' \ | column -t @@ -42,7 +42,7 @@ function create-report() { function run-gocloc-natively() { - gocloc --output-type=json . > cloc-report.tmp.json + gocloc --output-type=json . > lines-of-code-report.tmp.json } function run-gocloc-in-docker() { @@ -57,7 +57,7 @@ function run-gocloc-in-docker() { "$image" \ --output-type=json \ . \ - > cloc-report.tmp.json + > lines-of-code-report.tmp.json } function enrich-report() { @@ -74,9 +74,9 @@ function enrich-report() { # shellcheck disable=SC2086 jq \ '.creationInfo |= . + {"created":"'${build_datetime}'","repository":{"url":"'${git_url}'","branch":"'${git_branch}'","tags":['${git_tags}'],"commitHash":"'${git_commit_hash}'"},"pipeline":{"id":'${pipeline_run_id}',"number":'${pipeline_run_number}',"attempt":'${pipeline_run_attempt}'}}' \ - cloc-report.tmp.json \ - > cloc-report.json - rm -f cloc-report.tmp.json + lines-of-code-report.tmp.json \ + > lines-of-code-report.json + rm -f lines-of-code-report.tmp.json } # ============================================================================== diff --git a/scripts/reports/generate-sbom.sh b/scripts/reports/create-sbom-report.sh similarity index 98% rename from scripts/reports/generate-sbom.sh rename to scripts/reports/create-sbom-report.sh index 117c1cf0..1ed735a7 100755 --- a/scripts/reports/generate-sbom.sh +++ b/scripts/reports/create-sbom-report.sh @@ -10,7 +10,7 @@ set -euo pipefail # it in a Docker container. # # Usage: -# $ [options] ./generate-sbom.sh +# $ [options] ./create-sbom-report.sh # # Options: # BUILD_DATETIME=%Y-%m-%dT%H:%M:%S%z # Build datetime, default is `date -u +'%Y-%m-%dT%H:%M:%S%z'` diff --git a/scripts/reports/scan-vulnerabilities.sh b/scripts/reports/scan-vulnerabilities.sh index ef4e5020..eb68d4b5 100755 --- a/scripts/reports/scan-vulnerabilities.sh +++ b/scripts/reports/scan-vulnerabilities.sh @@ -17,7 +17,7 @@ set -euo pipefail # VERBOSE=true # Show all the executed commands, default is `false` # # Depends on: -# $ ./generate-sbom.sh +# $ ./create-sbom-report.sh # ============================================================================== From cd2e267b27a6739a7585cd0648600968e00c2e7a Mon Sep 17 00:00:00 2001 From: Dan Stefaniuk Date: Tue, 17 Oct 2023 16:21:28 +0100 Subject: [PATCH 5/5] Address peer review feedback --- docs/developer-guides/Bash_and_Make.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/developer-guides/Bash_and_Make.md b/docs/developer-guides/Bash_and_Make.md index 3bcd7753..778e4529 100644 --- a/docs/developer-guides/Bash_and_Make.md +++ b/docs/developer-guides/Bash_and_Make.md @@ -140,7 +140,7 @@ VERBOSE=1 scripts/shellscript-linter.sh ### Scripts -Most scripts provided with this repository template can utilise tools installed on your `PATH` if they are available or run them from within a Docker container. To force a script to use Docker, the `FORCE_USE_DOCKER` variable is provided. Here is an example of how to use it: +Most scripts provided with this repository template can utilise tools installed on your `PATH` if they are available or run them from within a Docker container. To force a script to use Docker, the `FORCE_USE_DOCKER` variable is provided. This feature increases configurability of the development environment, allowing you to use custom tooling by default if present on the command-line path. Here is an example of how to use it: ```shell FORCE_USE_DOCKER=1 scripts/shellscript-linter.sh