Rewrite healthcheck in bash, and implement separate _nohealthcheck tagged images as-per issue #43

.github/workflows/docker.yml

@@ -2,9 +2,13 @@ name: Docker
 
 on:
   push:
-    branches: [ master ]
+    branches:
+      - master
+      - dev
   pull_request:
-    branches: [ master ]
+    branches: 
+      - master
+      - dev
 
 jobs:
 

.github/workflows/linting.yml

@@ -2,9 +2,13 @@ name: Linting
 
 on:
   push:
-    branches: [ master ]
+    branches:
+      - master
+      - dev
   pull_request:
-    branches: [ master ]
+    branches:
+      - master
+      - dev
 
 jobs:
 

Dockerfile

@@ -240,10 +240,10 @@ RUN set -x && \
     echo "========== Done! =========="
 
 COPY etc/ /etc/
-COPY healthcheck.py /healthcheck.py
+COPY healthcheck.sh /healthcheck.sh
 
 EXPOSE 30104/tcp 8080/tcp 30001/tcp 30002/tcp 30003/tcp 30004/tcp 30005/tcp
 
 ENTRYPOINT [ "/init" ]
 
-HEALTHCHECK --start-period=30s CMD /healthcheck.py
+HEALTHCHECK --start-period=300s --interval=300s CMD /healthcheck.sh

README.md

@@ -13,6 +13,7 @@ For more information on what PiAware is, see here: [FlightAware - PiAware](https
 * `latest` should always contain the latest released versions of `rtl-sdr`, `bladeRF`, `tcllauncher`, `tcllib`, `piaware`, `dump1090`, `mlat-client`, `SoapySDR` and `dump978`. This image is built nightly from the [`master` branch](https://github.com/mikenye/docker-piaware/tree/master) [`Dockerfile`](https://github.com/mikenye/docker-piaware/blob/master/Dockerfile) for all supported architectures.
 * `development` ([`dev` branch](https://github.com/mikenye/docker-piaware/tree/master), [`Dockerfile`](https://github.com/mikenye/docker-piaware/blob/master/Dockerfile), `amd64` architecture only, built on commit, not recommended for production)
 * Specific version and architecture tags are available if required, however these are not regularly updated. It is generally recommended to run `latest`.
+* There are also `latest` and version-specific tags appended with `_nohealthcheck` where the container healthchecks have been excluded from the image build. See [issue #43](https://github.com/mikenye/docker-piaware/issues/43).
 
 ### Tags & Versions
 

buildx.sh

@@ -17,3 +17,13 @@ VERSION=$(docker run --rm --entrypoint cat ${REPO}/${IMAGE}:latest /VERSIONS | g
 
 # Build & push version-specific
 docker buildx build -t "${REPO}/${IMAGE}:${VERSION}" --compress --push --platform "${PLATFORMS}" .
+
+# BUILD NOHEALTHCHECK VERSION
+# Modify dockerfile to remove healthcheck
+sed '/^HEALTHCHECK /d' < Dockerfile > Dockerfile.nohealthcheck
+
+# Build & push latest
+docker buildx build -f Dockerfile.nohealthcheck -t ${REPO}/${IMAGE}:latest_nohealthcheck --compress --push --platform "${PLATFORMS}" .
+
+# Build & push version-specific
+docker buildx build -f Dockerfile.nohealthcheck -t "${REPO}/${IMAGE}:${VERSION}_nohealthcheck" --compress --push --platform "${PLATFORMS}" .

etc/services.d/piaware/log/run

@@ -4,4 +4,4 @@
 s6-envuidgid nobody
 s6-applyuidgid -U
 
-s6-log -bp 1 n5 s1000000 S10000000 T /var/log/piaware
+s6-log -bp 1 n5 s400000 S10000000 T /var/log/piaware

healthcheck.sh

@@ -0,0 +1,120 @@
+#!/usr/bin/with-contenv bash
+# shellcheck shell=bash
+
+# Globals
+S6_SERVICE_ROOT="/run/s6/services"
+STR_HEALTHY="OK"
+STR_UNHEALTHY="UNHEALTHY"
+
+EXITCODE=0
+
+function check_service_deathtally () {
+    local service_name
+    service_name=${1}
+
+    # build service path
+    local service_path
+    service_path="${S6_SERVICE_ROOT%/}/${service_name}"
+
+    # ensure service path exists
+    if [[ -d "$service_path" ]]; then
+
+        # get service death tally since last check
+        local service_deathtally
+        service_deathtally=$(s6-svdt "${service_path}" | wc -l)
+
+        # clear death tally
+        s6-svdt-clear "${service_path}"
+
+        # print the first part of the text
+        echo -n "\"${service_name}\" death tally since last check: ${service_deathtally}"
+
+        # if healthy/unhealthy...
+        if [[ "$service_deathtally" -gt 0 ]]; then
+            echo ": $STR_UNHEALTHY"
+            EXITCODE=1
+        else
+            echo ": $STR_HEALTHY"
+        fi
+    else
+
+        # if service directory doesn't exist, throw an error
+        echo "ERROR: service path \"$service_path\" does not exist!"
+        EXITCODE=1
+    fi
+}
+
+# MAIN
+
+set -o pipefail
+
+# check service death tallys
+check_service_deathtally 'beastproxy'
+check_service_deathtally 'beastrelay'
+check_service_deathtally 'dump1090'
+check_service_deathtally 'piaware'
+check_service_deathtally 'skyaware'
+
+# run piaware-status and store output
+PIAWARE_STATUS=$(piaware-status)
+
+# tests on piaware-status output
+if [[ "$(echo "${PIAWARE_STATUS}" | grep -c "PiAware master process (piaware) is running")" -lt 1 ]]; then
+    echo "piaware-status reports: PiAware master process (piaware) is NOT running: $STR_UNHEALTHY"
+    EXITCODE=1
+else
+    echo "piaware-status reports: PiAware master process (piaware) is running: $STR_HEALTHY"
+fi
+if [[ "$(echo "${PIAWARE_STATUS}" | grep -c "dump1090 is NOT producing data on")" -gt 0 ]]; then
+    echo "piaware-status reports: dump1090 is NOT producing data: $STR_UNHEALTHY"
+    EXITCODE=1
+else
+    echo "piaware-status reports: dump1090 is producing data: $STR_HEALTHY"
+fi
+if [[ "$(echo "${PIAWARE_STATUS}" | grep -c "piaware is connected to FlightAware")" -lt 1 ]]; then
+    echo "piaware is NOT conneceted to FlightAware: $STR_UNHEALTHY"
+    EXITCODE=1
+else
+    echo "piaware is conneceted to FlightAware: $STR_HEALTHY"
+fi
+
+# ensure we're sending data to FA
+DATETIME_NOW=$(date +%s)
+# find last log entry reporting messages sent to FA
+LOG_MSGS_SENT_LATEST=$(tail -100 /var/log/piaware/current | grep "msgs sent to FlightAware" | tail -1)
+if [[ -z "$LOG_MSGS_SENT_LATEST" ]]; then
+    echo "Logs indicate no msgs sent to FlightAware: $STR_UNHEALTHY"
+    EXITCODE=1
+else
+    # get date of log entry
+    LOG_MSGS_SENT_LATEST_DATETIME=$(date --date="$(echo "$LOG_MSGS_SENT_LATEST" | grep -oP '^\d{4}\-\d{2}\-\d{2} \d{2}\:\d{2}\:\d{2}')" +%s)
+    if [[ -z "$LOG_MSGS_SENT_LATEST_DATETIME" ]]; then
+        echo "Cannot determine date/time of last log entry of msgs sent to FlightAware: $STR_UNHEALTHY"
+        EXITCODE=1
+    else
+        # make sure last entry is less than 5 minutes old (plus extra minute grace period)
+        TIMEDELTA=$((DATETIME_NOW - LOG_MSGS_SENT_LATEST_DATETIME))
+        if [[ "$TIMEDELTA" -gt 360 ]]; then
+            echo "Logs indicate no msgs sent to FlightAware in past 5 minutes: $STR_UNHEALTHY"
+            EXITCODE=1
+        else
+            # get number of messages in last 5 min
+            LOG_MSGS_SENT_LATEST_NUM=$(echo "$LOG_MSGS_SENT_LATEST" | grep -oP '\(\d+ in last 5m\)' | cut -d '(' -f 2 | cut -d ' ' -f 1 | tr -d ' ')
+            if [[ -z "$LOG_MSGS_SENT_LATEST_NUM" ]]; then
+                echo "Cannot determine number of messages sent to FlightAware in last 5m: $STR_UNHEALTHY"
+                EXITCODE=1
+            else
+                echo -n "Logs indicate $LOG_MSGS_SENT_LATEST_NUM msgs sent to FlightAware in past 5 minutes:"
+                # make sure LOG_MSGS_SENT_LATEST_NUM is positive
+                if [[ "$LOG_MSGS_SENT_LATEST_NUM" -lt 1 ]]; then
+                    echo " $STR_UNHEALTHY"
+                    EXITCODE=1
+                else
+                    echo " $STR_HEALTHY"
+                fi
+            fi
+        fi
+    fi
+fi
+
+exit "$EXITCODE"