Refactor: Shell scripts and CI/CD pipeline

nhs-england-tools · Sep 9, 2023 · fac3252 · fac3252
1 parent bced9b0
commit fac3252
Show file tree

Hide file tree

Showing 10 changed files with 109 additions and 51 deletions.
diff --git a/.github/workflows/cicd-1-pull-request.yaml b/.github/workflows/cicd-1-pull-request.yaml
@@ -1,4 +1,4 @@
-name: "CI/CD Pull Request"
+name: "CI/CD pull request"
 
 # The total recommended execution time for the "CI/CD Pull Request" workflow is around 20 minutes.
 

diff --git a/.github/workflows/cicd-2-publish.yaml b/.github/workflows/cicd-2-publish.yaml
@@ -77,3 +77,19 @@ jobs:
       #     asset_path: ./*
       #     asset_name: repository-template-${{ needs.metadata.outputs.version }}.tar.gz
       #     asset_content_type: "application/gzip"
+  success:
+    runs-on: ubuntu-latest
+    needs: [publish]
+    steps:
+      - name: "Check prerequisites for notification"
+        id: check
+        run: echo "secret_exist=${{ secrets.TEAMS_NOTIFICATION_WEBHOOK_URL != '' }}" >> $GITHUB_OUTPUT
+      - name: "Notify on build completion"
+        if: steps.check.outputs.secret_exist == 'true'
+        uses: nhs-england-tools/[email protected]
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          teams-webhook-url: ${{ secrets.TEAMS_NOTIFICATION_WEBHOOK_URL }}
+          message-title: "Notification title"
+          message-text: "This is a notification body"
+          link: ${{ github.event.pull_request.html_url }}
diff --git a/.github/workflows/cicd-3-deploy.yaml b/.github/workflows/cicd-3-deploy.yaml
@@ -56,3 +56,19 @@ jobs:
       - name: "Checkout code"
         uses: actions/checkout@v4
   # TODO: More jobs or/and steps here
+  # success:
+  #   runs-on: ubuntu-latest
+  #   needs: [deploy]
+  #   steps:
+  #     - name: "Check prerequisites for notification"
+  #       id: check
+  #       run: echo "secret_exist=${{ secrets.TEAMS_NOTIFICATION_WEBHOOK_URL != '' }}" >> $GITHUB_OUTPUT
+  #     - name: "Notify on build completion"
+  #       if: steps.check.outputs.secret_exist == 'true'
+  #       uses: nhs-england-tools/[email protected]
+  #       with:
+  #         github-token: ${{ secrets.GITHUB_TOKEN }}
+  #         teams-webhook-url: ${{ secrets.TEAMS_NOTIFICATION_WEBHOOK_URL }}
+  #         message-title: "Notification title"
+  #         message-text: "This is a notification body"
+  #         link: ${{ github.event.pull_request.html_url }}
diff --git a/.github/workflows/stage-3-build.yaml b/.github/workflows/stage-3-build.yaml
@@ -65,19 +65,3 @@ jobs:
         run: |
           echo "Uploading artefact 2 ..."
           # TODO: Use either action/cache or action/upload-artifact
-  success:
-    runs-on: ubuntu-latest
-    needs: [artefact-1, artefact-2]
-    steps:
-      - name: "Check prerequisites for notification"
-        id: check
-        run: echo "secret_exist=${{ secrets.TEAMS_NOTIFICATION_WEBHOOK_URL != '' }}" >> $GITHUB_OUTPUT
-      - name: "Notify on build completion"
-        if: steps.check.outputs.secret_exist == 'true'
-        uses: nhs-england-tools/[email protected]
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          teams-webhook-url: ${{ secrets.TEAMS_NOTIFICATION_WEBHOOK_URL }}
-          message-title: "Notification title"
-          message-text: "This is a notification body"
-          link: ${{ github.event.pull_request.html_url }}
diff --git a/.github/workflows/stage-4-acceptance.yaml b/.github/workflows/stage-4-acceptance.yaml
@@ -53,7 +53,8 @@ jobs:
     needs: environment-set-up
     timeout-minutes: 10
     steps:
-      - uses: actions/checkout@v4
+      - name: "Checkout code"
+        uses: actions/checkout@v4
       - name: "Run contract test"
         run: |
           make test-contract
@@ -65,7 +66,8 @@ jobs:
     needs: environment-set-up
     timeout-minutes: 10
     steps:
-      - uses: actions/checkout@v4
+      - name: "Checkout code"
+        uses: actions/checkout@v4
       - name: "Run security test"
         run: |
           make test-security
@@ -77,7 +79,8 @@ jobs:
     needs: environment-set-up
     timeout-minutes: 10
     steps:
-      - uses: actions/checkout@v4
+      - name: "Checkout code"
+        uses: actions/checkout@v4
       - name: "Run UI test"
         run: |
           make test-ui
@@ -89,7 +92,8 @@ jobs:
     needs: environment-set-up
     timeout-minutes: 10
     steps:
-      - uses: actions/checkout@v4
+      - name: "Checkout code"
+        uses: actions/checkout@v4
       - name: "Run UI performance test"
         run: |
           make test-ui-performance

diff --git a/scripts/docker/docker.lib.sh b/scripts/docker/docker.lib.sh
@@ -1,4 +1,5 @@
 #!/bin/bash
+# shellcheck disable=SC2155
 
 # WARNING: Please, DO NOT edit this file! It is maintained in the Repository Template (https://github.com/nhs-england-tools/repository-template). Raise a PR instead.
 
@@ -22,6 +23,7 @@ set -euo pipefail
 function docker-build() {
 
   local dir=${dir:-$PWD}
+
   _create-effective-dockerfile
   version-create-effective-file
   docker build \
@@ -35,14 +37,14 @@ function docker-build() {
     --build-arg GIT_BRANCH="$(_get-git-branch-name)" \
     --build-arg GIT_COMMIT_HASH="$(git rev-parse --short HEAD)" \
     --build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%S%z")" \
-    --build-arg BUILD_VERSION="$(_get-version)" \
-    --tag "${DOCKER_IMAGE}:$(_get-version)" \
+    --build-arg BUILD_VERSION="$(_get-effective-version)" \
+    --tag "${DOCKER_IMAGE}:$(_get-effective-version)" \
     --rm \
     --file "${dir}/Dockerfile.effective" \
     .
   # Tag the image with all the stated versions, see the documentation for more details
-  for version in $(_get-all-versions) latest; do
-    docker tag "${DOCKER_IMAGE}:$(_get-version)" "${DOCKER_IMAGE}:${version}"
+  for version in $(_get-all-effective-versions) latest; do
+    docker tag "${DOCKER_IMAGE}:$(_get-effective-version)" "${DOCKER_IMAGE}:${version}"
   done
   docker rmi --force "$(docker images | grep "<none>" | awk '{print $3}')" 2> /dev/null ||:
 }
@@ -56,10 +58,11 @@ function docker-build() {
 function docker-check-test() {
 
   local dir=${dir:-$PWD}
+
   # shellcheck disable=SC2086,SC2154
   docker run --rm --platform linux/amd64 \
     ${args:-} \
-    "${DOCKER_IMAGE}:$(_get-version)" 2>/dev/null \
+    "${DOCKER_IMAGE}:$(_get-effective-version)" 2>/dev/null \
     ${cmd:-} \
   | grep -q "${check}" && echo PASS || echo FAIL
 }
@@ -72,10 +75,11 @@ function docker-check-test() {
 function docker-run() {
 
   local dir=${dir:-$PWD}
+
   # shellcheck disable=SC2086
   docker run --rm --platform linux/amd64 \
     ${args:-} \
-    "${DOCKER_IMAGE}:$(dir="$dir" _get-version)" \
+    "${DOCKER_IMAGE}:$(dir="$dir" _get-effective-version)" \
     ${cmd:-}
 }
 
@@ -85,8 +89,9 @@ function docker-run() {
 function docker-push() {
 
   local dir=${dir:-$PWD}
+
   # Push all the image tags based on the stated versions, see the documentation for more details
-  for version in $(dir="$dir" _get-all-versions) latest; do
+  for version in $(dir="$dir" _get-all-effective-versions) latest; do
     docker push "${DOCKER_IMAGE}:${version}"
   done
 }
@@ -97,7 +102,8 @@ function docker-push() {
 function docker-clean() {
 
   local dir=${dir:-$PWD}
-  for version in $(dir="$dir" _get-all-versions) latest; do
+
+  for version in $(dir="$dir" _get-all-effective-versions) latest; do
     docker rmi "${DOCKER_IMAGE}:${version}" > /dev/null 2>&1 ||:
   done
   rm -f \
@@ -112,10 +118,12 @@ function docker-clean() {
 function version-create-effective-file() {
 
   local dir=${dir:-$PWD}
-  build_datetime=${BUILD_DATETIME:-$(date -u +'%Y-%m-%dT%H:%M:%S%z')}
-  if [ -f "$dir/VERSION" ]; then
+  local version_file="$dir/VERSION"
+  local build_datetime=${BUILD_DATETIME:-$(date -u +'%Y-%m-%dT%H:%M:%S%z')}
+
+  if [ -f "$version_file" ]; then
     # shellcheck disable=SC2002
-    cat "$dir/VERSION" | \
+    cat "$version_file" | \
       sed "s/\(\${yyyy}\|\$yyyy\)/$(date --date="${build_datetime}" -u +"%Y")/g" | \
       sed "s/\(\${mm}\|\$mm\)/$(date --date="${build_datetime}" -u +"%m")/g" | \
       sed "s/\(\${dd}\|\$dd\)/$(date --date="${build_datetime}" -u +"%d")/g" | \
@@ -132,18 +140,18 @@ function version-create-effective-file() {
 
 # Retrieve the Docker image version from the '.tool-versions' file and pull the
 # image if required. This function is to be used in conjunction with the
-# external images and it prevents Docker from downloading an image each time it
-# is used, since the digest is not stored locally for compressed images. To
-# optimise, the solution is to pull the image using its digest and then tag it,
-# checking this tag for existence for any subsequent use.
+# external images and it prevents Docker from downloading an image each time it
+# is used, since the digest is not stored locally for compressed images. To
+# optimise, the solution is to pull the image using its digest and then tag it,
+# checking this tag for existence for any subsequent use.
 # Arguments (provided as environment variables):
 #   name=[full name of the Docker image]
 # shellcheck disable=SC2001
 function docker-get-image-version-and-pull() {
 
   # Get the image full version from the '.tool-versions' file
-  versions_file="$(git rev-parse --show-toplevel)/.tool-versions"
-  version="latest"
+  local versions_file="$(git rev-parse --show-toplevel)/.tool-versions"
+  local version="latest"
   if [ -f "$versions_file" ]; then
     line=$(grep "docker/${name} " "$versions_file" | sed "s/^#\s*//; s/\s*#.*$//")
     [ -n "$line" ] && version=$(echo "$line" | awk '{print $2}')
@@ -155,8 +163,8 @@ function docker-get-image-version-and-pull() {
   #   version="1.2.3@sha256:hash"
   #   tag="1.2.3"
   #   digest="sha256:hash"
-  tag="$(echo "$version" | sed 's/@.*$//')"
-  digest="$(echo "$version" | sed 's/^.*@//')"
+  local tag="$(echo "$version" | sed 's/@.*$//')"
+  local digest="$(echo "$version" | sed 's/^.*@//')"
 
   # Check if the image exists locally already
   if ! docker images | awk '{ print $1 ":" $2 }' | grep "^${name}:${tag}$"; then
@@ -188,6 +196,7 @@ function docker-get-image-version-and-pull() {
 function _create-effective-dockerfile() {
 
   local dir=${dir:-$PWD}
+
   cp "${dir}/Dockerfile" "${dir}/Dockerfile.effective"
   _replace-image-latest-by-specific-version
   _append-metadata
@@ -199,7 +208,10 @@ function _create-effective-dockerfile() {
 function _replace-image-latest-by-specific-version() {
 
   local dir=${dir:-$PWD}
-  versions_file=$(git rev-parse --show-toplevel)/.tool-versions
+  local versions_file=$(git rev-parse --show-toplevel)/.tool-versions
+  local dockerfile="${dir}/Dockerfile.effective"
+  local build_datetime=${BUILD_DATETIME:-$(date -u +'%Y-%m-%dT%H:%M:%S%z')}
+
   if [ -f "$versions_file" ]; then
     # First, list the entries specific for Docker to take precedence, then the rest
     content=$(grep " docker/" "$versions_file"; grep -v " docker/" "$versions_file")
@@ -208,9 +220,24 @@ function _replace-image-latest-by-specific-version() {
       line=$(echo "$line" | sed "s/^#\s*//; s/\s*#.*$//" | sed "s;docker/;;")
       name=$(echo "$line" | awk '{print $1}')
       version=$(echo "$line" | awk '{print $2}')
-      sed -i "s;FROM ${name}:latest;FROM ${name}:${version};g" "${dir}/Dockerfile.effective"
+      sed -i "s;\(FROM .*\) ${name}:latest;\1 ${name}:${version};g" "$dockerfile"
     done
   fi
+
+  if [ -f "$dockerfile" ]; then
+    # shellcheck disable=SC2002
+    cat "$dockerfile" | \
+      sed "s/\(\${yyyy}\|\$yyyy\)/$(date --date="${build_datetime}" -u +"%Y")/g" | \
+      sed "s/\(\${mm}\|\$mm\)/$(date --date="${build_datetime}" -u +"%m")/g" | \
+      sed "s/\(\${dd}\|\$dd\)/$(date --date="${build_datetime}" -u +"%d")/g" | \
+      sed "s/\(\${HH}\|\$HH\)/$(date --date="${build_datetime}" -u +"%H")/g" | \
+      sed "s/\(\${MM}\|\$MM\)/$(date --date="${build_datetime}" -u +"%M")/g" | \
+      sed "s/\(\${SS}\|\$SS\)/$(date --date="${build_datetime}" -u +"%S")/g" | \
+      sed "s/\(\${hash}\|\$hash\)/$(git rev-parse --short HEAD)/g" \
+    > "$dockerfile.tmp"
+  mv "$dockerfile.tmp" "$dockerfile"
+  fi
+
   # Do not ignore the issue if 'latest' is used in the effective image
   sed -Ei "/# hadolint ignore=DL3007$/d" "${dir}/Dockerfile.effective"
 }
@@ -221,6 +248,7 @@ function _replace-image-latest-by-specific-version() {
 function _append-metadata() {
 
   local dir=${dir:-$PWD}
+
   cat \
     "$dir/Dockerfile.effective" \
     "$(git rev-parse --show-toplevel)/scripts/docker/Dockerfile.metadata" \
@@ -231,26 +259,29 @@ function _append-metadata() {
 # Print top Docker image version.
 # Arguments (provided as environment variables):
 #   dir=[path to the image directory where the Dockerfile is located, default is '.']
-function _get-version() {
+function _get-effective-version() {
 
   local dir=${dir:-$PWD}
+
   head -n 1 "${dir}/.version" 2> /dev/null ||:
 }
 
 # Print all Docker image versions.
 # Arguments (provided as environment variables):
 #   dir=[path to the image directory where the Dockerfile is located, default is '.']
-function _get-all-versions() {
+function _get-all-effective-versions() {
 
   local dir=${dir:-$PWD}
+
   cat "${dir}/.version" 2> /dev/null ||:
 }
 
 # Print Git branch name. Check the GitHub variables first and then the local Git
-# repo.
+# repo.
 function _get-git-branch-name() {
 
-  branch_name=$(git rev-parse --abbrev-ref HEAD)
+  local branch_name=$(git rev-parse --abbrev-ref HEAD)
+
   if [ -n "${GITHUB_HEAD_REF:-}" ]; then
     branch_name=$GITHUB_HEAD_REF
   elif [ -n "${GITHUB_REF:-}" ]; then

diff --git a/scripts/docker/docker.mk b/scripts/docker/docker.mk
@@ -7,6 +7,9 @@
 docker-build: # Build Docker image - optional: dir=[path to the Dockerfile to use, default is '.']
 	make _docker cmd="build"
 
+docker-push: # Push Docker image - optional: dir=[path to the image directory where the Dockerfile is located, default is '.']
+	make _docker cmd="push"
+
 clean:: # Remove Docker resources - optional: dir=[path to the image directory where the Dockerfile is located, default is '.']
 	make _docker cmd="clean"
 
@@ -68,5 +71,6 @@ ${VERBOSE}.SILENT: \
 	docker-example-clean \
 	docker-example-lint \
 	docker-example-run \
+	docker-push \
 	docker-shellscript-lint \
 	docker-test-suite-run \
diff --git a/scripts/docker/tests/docker.test.sh b/scripts/docker/tests/docker.test.sh
@@ -32,10 +32,11 @@ function main() {
     test-docker-run \
     test-docker-clean \
   )
-  status=0
+  local status=0
   for test in "${tests[@]}"; do
     {
       echo -n "$test"
+      # shellcheck disable=SC2015
       $test && echo " PASS" || { echo " FAIL"; ((status++)); }
     }
   done
@@ -65,7 +66,7 @@ function test-docker-build() {
   # Act
   docker-build > /dev/null 2>&1
   # Assert
-  docker image inspect "${DOCKER_IMAGE}:$(_get-version)" > /dev/null 2>&1 && return 0 || return 1
+  docker image inspect "${DOCKER_IMAGE}:$(_get-effective-version)" > /dev/null 2>&1 && return 0 || return 1
 }
 
 function test-docker-version() {
@@ -75,6 +76,7 @@ function test-docker-version() {
   # Act
   version-create-effective-file
   # Assert
+  # shellcheck disable=SC2002
   (
       cat .version | grep -q "20230904-" &&
       cat .version | grep -q "2023.09.04-" &&
@@ -106,7 +108,7 @@ function test-docker-run() {
 function test-docker-clean() {
 
   # Arrange
-  version="$(_get-version)"
+  version="$(_get-effective-version)"
   # Act
   docker-clean
   # Assert

diff --git a/scripts/shellscript-linter.sh b/scripts/shellscript-linter.sh
@@ -46,7 +46,8 @@ function docker-run-shellcheck() {
   # shellcheck disable=SC1091
   source ./scripts/docker/docker.lib.sh
 
-  image=$(name=koalaman/shellcheck docker-get-image-version-and-pull)
+  # shellcheck disable=SC2155
+  local image=$(name=koalaman/shellcheck docker-get-image-version-and-pull)
   # shellcheck disable=SC2001
   docker run --rm --platform linux/amd64 \
     --volume "$PWD:/workdir" \