fix(mae-consumer): fix regression on base64 encoding (#6061)

Pull request #5827 introduced a regression by removing coreutils
from the mae-consumer Dockerfile (coreutils was added in #3723).
This broke the base64 call in the startup script s.th. the Elasticsearch
auth header will not be correctly set when username and password are
provided.

To make sure that the startup script fails on these errors in the
future, set "-euo pipefail" which lets the bash script fail on errors
and unset variables.
Also refactor the startup script to make it more stable and readable.

Co-authored-by: Pedro Silva <[email protected]>

docker/datahub-mae-consumer/Dockerfile

@@ -14,7 +14,7 @@ RUN apk --no-cache --update-cache --available upgrade \
     else \
       echo >&2 "Unsupported architecture $(arch)" ; exit 1; \
     fi \
-    && apk --no-cache add tar curl bash \
+    && apk --no-cache add tar curl bash coreutils \
     && apk --no-cache add openjdk11-jre --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community \
     && wget --no-verbose https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.4.1/opentelemetry-javaagent-all.jar \
     && wget --no-verbose https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.16.1/jmx_prometheus_javaagent-0.16.1.jar -O jmx_prometheus_javaagent.jar \

docker/datahub-mae-consumer/start.sh

@@ -1,61 +1,46 @@
 #!/bin/bash
+set -euo pipefail
 
 # Add default URI (http) scheme if needed
-if ! echo $NEO4J_HOST | grep -q "://" ; then
-    NEO4J_HOST="http://$NEO4J_HOST"
+if [[ -n ${NEO4J_HOST:-} ]] && [[ ${NEO4J_HOST} != *"://"* ]]; then
+  NEO4J_HOST="http://$NEO4J_HOST"
 fi
 
-if [[ ! -z $ELASTICSEARCH_USERNAME ]] && [[ -z $ELASTICSEARCH_AUTH_HEADER ]]; then
+if [[ -n ${ELASTICSEARCH_USERNAME:-} ]] && [[ -z ${ELASTICSEARCH_AUTH_HEADER:-} ]]; then
   AUTH_TOKEN=$(echo -ne "$ELASTICSEARCH_USERNAME:$ELASTICSEARCH_PASSWORD" | base64 --wrap 0)
   ELASTICSEARCH_AUTH_HEADER="Authorization:Basic $AUTH_TOKEN"
 fi
 
 # Add default header if needed
-if [[ -z $ELASTICSEARCH_AUTH_HEADER ]]; then
-    ELASTICSEARCH_AUTH_HEADER="Accept: */*"
-fi
+: "${ELASTICSEARCH_AUTH_HEADER="Accept: */*"}"
 
-if [[ $ELASTICSEARCH_USE_SSL == true ]]; then
+if [[ ${ELASTICSEARCH_USE_SSL:-false} == true ]]; then
     ELASTICSEARCH_PROTOCOL=https
 else
     ELASTICSEARCH_PROTOCOL=http
 fi
 
-WAIT_FOR_KAFKA=""
-if [[ $SKIP_KAFKA_CHECK != true ]]; then
-  WAIT_FOR_KAFKA=" -wait tcp://$(echo $KAFKA_BOOTSTRAP_SERVER | sed 's/,/ -wait tcp:\/\//g') "
+dockerize_args=("-timeout" "240s")
+if [[ ${SKIP_KAFKA_CHECK:-false} != true ]]; then
+  IFS=',' read -ra KAFKAS <<< "$KAFKA_BOOTSTRAP_SERVER"
+  for i in "${KAFKAS[@]}"; do
+    dockerize_args+=("-wait" "tcp://$i")
+  done
 fi
-
-WAIT_FOR_ELASTICSEARCH=""
-if [[ $SKIP_ELASTICSEARCH_CHECK != true ]]; then
-  WAIT_FOR_ELASTICSEARCH=" -wait $ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT -wait-http-header \"$ELASTICSEARCH_AUTH_HEADER\""
+if [[ ${SKIP_ELASTICSEARCH_CHECK:-false} != true ]]; then
+  dockerize_args+=("-wait" "$ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT" "-wait-http-header" "$ELASTICSEARCH_AUTH_HEADER")
 fi
-
-WAIT_FOR_NEO4J=""
-if [[ $GRAPH_SERVICE_IMPL != elasticsearch ]] && [[ $SKIP_NEO4J_CHECK != true ]]; then
-  WAIT_FOR_NEO4J=" -wait $NEO4J_HOST "
+if [[ ${GRAPH_SERVICE_IMPL:-} != elasticsearch ]] && [[ ${SKIP_NEO4J_CHECK:-false} != true ]]; then
+  dockerize_args+=("-wait" "$NEO4J_HOST")
 fi
 
-OTEL_AGENT=""
-if [[ $ENABLE_OTEL == true ]]; then
-  OTEL_AGENT="-javaagent:opentelemetry-javaagent-all.jar "
+JDK_JAVA_OPTIONS="${JDK_JAVA_OPTIONS:-}${JAVA_OPTS:+ JAVA_OPTS}${JMX_OPTS:+ JMX_OPTS}"
+if [[ ${ENABLE_OTEL:-false} == true ]]; then
+  JDK_JAVA_OPTIONS="$JDK_JAVA_OPTIONS -javaagent:opentelemetry-javaagent-all.jar"
 fi
-
-PROMETHEUS_AGENT=""
-if [[ $ENABLE_PROMETHEUS == true ]]; then
-  PROMETHEUS_AGENT="-javaagent:jmx_prometheus_javaagent.jar=4318:/datahub/datahub-mae-consumer/scripts/prometheus-config.yaml "
+if [[ ${ENABLE_PROMETHEUS:-false} == true ]]; then
+  JDK_JAVA_OPTIONS="$JDK_JAVA_OPTIONS -javaagent:jmx_prometheus_javaagent.jar=4318:/datahub/datahub-mae-consumer/scripts/prometheus-config.yaml"
 fi
 
-COMMON="
-    $WAIT_FOR_KAFKA \
-    $WAIT_FOR_NEO4J \
-    -timeout 240s \
-    java $JAVA_OPTS $JMX_OPTS $OTEL_AGENT $PROMETHEUS_AGENT -jar /datahub/datahub-mae-consumer/bin/mae-consumer-job.jar
-"
-if [[ $SKIP_ELASTICSEARCH_CHECK != true ]]; then
-  exec dockerize \
-    -wait $ELASTICSEARCH_PROTOCOL://$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT -wait-http-header "$ELASTICSEARCH_AUTH_HEADER" \
-    $COMMON
-else
-  exec dockerize $COMMON
-fi
+export JDK_JAVA_OPTIONS
+exec dockerize "${dockerize_args[@]}" java -jar /datahub/datahub-mae-consumer/bin/mae-consumer-job.jar