[INF-93] Remove rsyslog implementation and build Logspout sidecar con…

…tainer (#3124)

* Build a custom Logspout container for SPs

* rm rsyslog

* stop tee'ing to logger

* separate one process per container

* missing audius_service envar

* mild fixes before revert

* Revert "breaking up dp container"

This reverts commit 74ad707.

* Revert "missing audius_service envar"

This reverts commit ecd0ea3.

* Revert "separate one process per container"

This reverts commit 2609fa0.

* rm last syslog calls

* use audius_loggly_token everywhere

* background job

* image tagging

* update build command

* Dockerfile update

* th

* kansas city shuffle

* fix building

* remove healthcheck

* fix Dockerfile ordering and loggly tagging

* fix circleci param

* fix shuffle

* set tags correctly

* sh not bash

* Add more log tags

* default logspout-tag

* quotes

* no logspout test

* true string

* dir

* mv logging/logspout logspout

* repo: logspout

* mv logging/logspout logspout

* remove redundant tagging

* add hostname, if it exists

* Update default_config.ini

* Update helpers.py

* gate rsyslog

* restore discovery-provider/scripts/rotate-log.sh

* chmod 755

* gate for dp

* gate rsyslog

* keep rsyslog integration with circleci

* fix newlines

* ordering

* restore dockerfiles

* add manual approval req for logstash image building

* use true/false instead of 0/1 for envars

* default local to DEBUG logs

* extract hostname from cn and dp nodes

* audius_discprov_loglevel_flask

* formatting

* Update README.md

* Update start.sh

* revert changes for identity service

* add additional comment

* address PR comment

* add README context

.circleci/config.yml

@@ -637,6 +637,9 @@ jobs:
     parameters:
       repo:
         type: string
+      logspout-tag:
+        type: string
+        default: ""
     steps:
       - checkout
       - setup_remote_docker:
@@ -645,10 +648,11 @@ jobs:
           name: Get tag or latest
           command: |
             # order of precendence for IMAGE tag is:
-            # 1. CIRCLE_TAG if defined,
-            # 2. Branch name if 'hotfix' in branch name (branch name passed in via $CIRCLE_BRANCH)
-            # 3. 'latest', which is the default
-            echo "export IMAGE_TAG=`[ $CIRCLE_TAG ] && echo $(echo $CIRCLE_TAG | cut -d@ -f3) || [[ "$CIRCLE_BRANCH" =~ (hotfix) ]] && echo $CIRCLE_BRANCH || echo "latest" `" | tee -a $BASH_ENV
+            # 1. $(head -n1 logging/logspout/Dockerfile) if << parameters.logspout-tag >> defined,
+            # 2. CIRCLE_TAG if defined,
+            # 3. Branch name if 'hotfix' in branch name (branch name passed in via $CIRCLE_BRANCH)
+            # 4. 'latest', which is the default
+            echo "export IMAGE_TAG=`[ << parameters.logspout-tag >> ] && echo $(head -n1 logspout/Dockerfile | cut -f 2 -d ':') || [ $CIRCLE_TAG ] && echo $(echo $CIRCLE_TAG | cut -d@ -f3) || [[ "$CIRCLE_BRANCH" =~ (hotfix) ]] && echo $CIRCLE_BRANCH || echo "latest" `" | tee -a $BASH_ENV
       - run:
           name: Docker login
           command: |
@@ -657,7 +661,8 @@ jobs:
           name: Docker build << parameters.repo >>
           command: |
             cd << parameters.repo >>
-            docker build -t audius/<< parameters.repo >>:$IMAGE_TAG -t audius/<< parameters.repo>>:$(git rev-parse HEAD) --build-arg git_sha=$(git rev-parse HEAD) --build-arg audius_loggly_disable=$audius_loggly_disable --build-arg audius_loggly_token=$audius_loggly_token --build-arg audius_loggly_tags=$audius_loggly_tags --build-arg BUILD_NUM=$CIRCLE_BUILD_NUM .
+            audius_loggly_token_b64=$(echo ${audius_loggly_token} | base64)
+            docker build -t audius/<< parameters.repo >>:$IMAGE_TAG -t audius/<< parameters.repo>>:$(git rev-parse HEAD) --build-arg git_sha=$(git rev-parse HEAD) --build-arg audius_loggly_disable=$audius_loggly_disable --build-arg audius_loggly_token=$audius_loggly_token --build-arg audius_loggly_tags=$audius_loggly_tags --build-arg BUILD_NUM=$CIRCLE_BUILD_NUM --build-arg audius_loggly_token=$audius_loggly_token_b64 .
             docker image prune --filter label=prune=true --filter label=build=$CIRCLE_BUILD_NUM --force
       - run:
           name: Docker push << parameters.repo >>
@@ -733,6 +738,15 @@ workflows:
           name: build-identity-service
           repo: identity-service
 
+      - hold-build-logspout:
+          type: approval
+      - docker-build-and-push:
+          name: build-logspout
+          repo: logspout
+          logspout-tag: "true"
+          requires:
+            - hold-build-logspout
+
       - test-solana-programs:
           name: test-solana-programs
       - test-solana-programs-anchor:

creator-node/scripts/start.sh

@@ -7,13 +7,18 @@ if [[ "$WAIT_HOSTS" != "" ]]; then
     /usr/bin/wait
 fi
 
-if [[ -z "$logglyDisable" ]]; then
-    if [[ -n "$logglyToken" ]]; then
-        logglyTags=$(echo $logglyTags | python3 -c "print(' '.join(f'tag=\\\\\"{i}\\\\\"' for i in input().split(',')))")
-        mkdir -p /var/spool/rsyslog
-        mkdir -p /etc/rsyslog.d
-        sed -i '1s|^|$MaxMessageSize 64k\n|' /etc/rsyslog.conf
-        cat >/etc/rsyslog.d/22-loggly.conf <<EOF
+# enable rsyslog if not explicitly disabled by audius-docker-compose
+: "${enableRsyslog:=true}"
+
+# $enableRsyslog should be true
+# $logglyDisable should be empty/null
+# $logglyToken should be a nonzero length string
+if $enableRsyslog && [[ -z "$logglyDisable" && -n "$logglyToken" ]]; then
+    logglyTags=$(echo $logglyTags | python3 -c "print(' '.join(f'tag=\\\\\"{i}\\\\\"' for i in input().split(',')))")
+    mkdir -p /var/spool/rsyslog
+    mkdir -p /etc/rsyslog.d
+    sed -i '1s|^|$MaxMessageSize 64k\n|' /etc/rsyslog.conf
+    cat >/etc/rsyslog.d/22-loggly.conf <<EOF
 \$WorkDirectory /var/spool/rsyslog # where to place spool files
 \$ActionQueueFileName fwdRule1   # unique name prefix for spool files
 \$ActionQueueMaxDiskSpace 1g    # 1gb space limit (use as much as possible)
@@ -25,8 +30,7 @@ template(name="LogglyFormat" type="string"
 # Send messages to Loggly over TCP using the template.
 action(type="omfwd" protocol="tcp" target="logs-01.loggly.com" port="514" template="LogglyFormat")
 EOF
-        rsyslogd
-    fi
+    rsyslogd
 fi
 
 if [ -z "$redisHost" ]; then

discovery-provider/compose/env/commonEnv.sh

@@ -3,3 +3,4 @@ export audius_delegate_private_key=AUDIUS_DELEGATE_PRIVATE_KEY
 echo $audius_delegate_owner_wallet
 echo $audius_delegate_private_key
 export COMPOSE_HTTP_TIMEOUT=200
+export audius_discprov_loglevel_flask=DEBUG

discovery-provider/default_config.ini

@@ -1,6 +1,6 @@
 [discprov]
 start_block = 0x0
-loglevel_flask = DEBUG
+loglevel_flask = INFO
 ; do not configure the log level here as this gets overridden by celery lib during setup
 ; set log level via command line in docker yml files instead
 ; loglevel_celery = INFO

discovery-provider/scripts/start.sh

@@ -1,12 +1,18 @@
 #!/bin/bash
 set -e
 
-mkdir -p /var/log
-mkdir -p /var/spool/rsyslog
-mkdir -p /etc/rsyslog.d
-
-if [[ -z "$audius_loggly_disable" ]]; then
-    if [[ -n "$audius_loggly_token" ]]; then
+# enable rsyslog if not explicitly disabled by audius-docker-compose
+: "${audius_enable_rsyslog:=true}"
+
+# $audius_enable_rsyslog should be true
+if $audius_enable_rsyslog; then
+    mkdir -p /var/log
+    mkdir -p /var/spool/rsyslog
+    mkdir -p /etc/rsyslog.d
+
+    # $logglyDisable should be empty/null
+    # $logglyToken should be a nonzero length string
+    if [[ -z "$audius_loggly_disable" && -n "$audius_loggly_token" ]]; then
         # use regex to extract domain in url (source: https://stackoverflow.com/a/2506635/8674706)
         audius_discprov_hostname=$(echo $audius_discprov_url | sed -e 's/[^/]*\/\/\([^@]*@\)\?\([^:/]*\).*/\2/')
 
@@ -34,15 +40,14 @@ if [[ -z "$audius_loggly_disable" ]]; then
 \$ActionResumeRetryCount -1         # infinite retries if host is down
 
 template(name="LogglyFormat" type="string"
- string="<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$audius_loggly_token@41058 $audius_loggly_tags] %msg%\n")
+string="<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$audius_loggly_token@41058 $audius_loggly_tags] %msg%\n")
 
 # Send messages to Loggly over TCP using the template.
 action(type="omfwd" protocol="tcp" target="logs-01.loggly.com" port="514" template="LogglyFormat")
 EOF
     fi
-fi
 
-cat >/etc/rsyslog.d/20-file.conf <<EOF
+    cat >/etc/rsyslog.d/20-file.conf <<EOF
 \$WorkDirectory /var/spool/rsyslog  # where to place spool files
 \$ActionQueueFileName file          # unique name prefix for spool files
 \$ActionQueueMaxDiskSpace 1g        # 1gb space limit (use as much as possible)
@@ -59,7 +64,8 @@ if \$programname == 'worker' then :omfile:\$worker_log
 if \$programname == 'beat' then   :omfile:\$beat_log
 EOF
 
-rsyslogd
+    rsyslogd
+fi
 
 if [ -z "$audius_redis_url" ]; then
     redis-server --daemonize yes
@@ -106,20 +112,16 @@ if [[ "$audius_discprov_dev_mode" == "true" ]]; then
         echo "Finished running migrations"
     fi
 
-    # filter tail to server/worker/beat logs with
-    # docker exec -it <container> tail -f /var/log/discprov-server.log
-    # docker exec -it <container> tail -f /var/log/discprov-worker.log
-    # docker exec -it <container> tail -f /var/log/discprov-beat.log
-    ./scripts/dev-server.sh 2>&1 | tee >(logger -t server) &
+    audius_service=server ./scripts/dev-server.sh 2>&1 | tee >(logger -t server) &
     if [[ "$audius_no_workers" != "true" ]] && [[ "$audius_no_workers" != "1" ]]; then
-        watchmedo auto-restart --directory ./ --pattern=*.py --recursive -- celery -A src.worker.celery worker --loglevel $audius_discprov_loglevel 2>&1 | tee >(logger -t worker) &
-        celery -A src.worker.celery beat --loglevel $audius_discprov_loglevel 2>&1 | tee >(logger -t beat) &
+        audius_service=worker watchmedo auto-restart --directory ./ --pattern=*.py --recursive -- celery -A src.worker.celery worker --loglevel $audius_discprov_loglevel 2>&1 | tee >(logger -t worker) &
+        audius_service=beat celery -A src.worker.celery beat --loglevel $audius_discprov_loglevel 2>&1 | tee >(logger -t beat) &
     fi
 else
-    ./scripts/prod-server.sh 2>&1 | tee >(logger -t server) &
+    audius_service=server ./scripts/prod-server.sh 2>&1 | tee >(logger -t server) &
     if [[ "$audius_no_workers" != "true" ]] && [[ "$audius_no_workers" != "1" ]]; then
-        celery -A src.worker.celery worker --loglevel $audius_discprov_loglevel 2>&1 | tee >(logger -t worker) &
-        celery -A src.worker.celery beat --loglevel $audius_discprov_loglevel 2>&1 | tee >(logger -t beat) &
+        audius_service=worker celery -A src.worker.celery worker --loglevel $audius_discprov_loglevel 2>&1 | tee >(logger -t worker) &
+        audius_service=beat celery -A src.worker.celery beat --loglevel $audius_discprov_loglevel 2>&1 | tee >(logger -t beat) &
     fi
 fi
 

discovery-provider/src/utils/helpers.py

@@ -205,10 +205,14 @@ def tuple_to_model_dictionary(t, model):
 
 
 log_format = {
-    "levelno": "levelno",
-    "level": "levelname",
+    "level": "levelno",
+    "logLevel": "levelname",
     "msg": "message",
     "timestamp": "asctime",
+    "pathname": "pathname",
+    "funcName": "funcName",
+    "lineno": "lineno",
+    "service": os.getenv("audius_service"),
 }
 
 formatter = JsonFormatter(log_format, ensure_ascii=False, mix_extra=True)

logspout/.gitignore

@@ -0,0 +1,2 @@
+.env
+

logspout/Dockerfile

@@ -0,0 +1,16 @@
+FROM gliderlabs/logspout:v3.2.14
+
+# ignores previously log on startup
+# could be lossy if the container restarts
+# could send double logs upon container restarts if we remove this
+ENV BACKLOG false
+
+ARG git_sha
+ENV GIT_SHA=${git_sha}
+
+ARG audius_loggly_token
+ENV audius_loggly_token ${audius_loggly_token}
+
+ENTRYPOINT []
+COPY start.sh /start.sh
+CMD /start.sh

logspout/README.md

@@ -0,0 +1,19 @@
+# Build Custom Audius Logspout Sidecar Container
+
+In the event we want to build the Logspout container by hand
+(for testing purposes, CircleCI is down, etc) run the following:
+
+```bash
+# .env contains: audius_loggly_token=xxx
+. .env
+
+LOGSPOUT_VERSION=$(head -n1 Dockerfile | cut -f 2 -d ':')
+[ ${audius_loggly_token} ] \
+    && audius_loggly_token_64=$(echo ${audius_loggly_token} | base64) \
+    && docker build \
+        -t audius/logspout:${LOGSPOUT_VERSION} \
+        --build-arg git_sha=$(git rev-parse HEAD) \
+        --build-arg audius_loggly_token=${audius_loggly_token_64} \
+        . \
+    && docker push audius/logspout:${LOGSPOUT_VERSION}
+```

logspout/build.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+# https://github.com/gliderlabs/logspout/blob/818dd8260e52d2c148280d86170bdf5267b5c637/build.sh
+# due to: https://github.com/gliderlabs/logspout/blob/818dd8260e52d2c148280d86170bdf5267b5c637/Dockerfile#L9-L11
+
+set -e
+apk add --update go build-base git mercurial ca-certificates
+cd /src
+go build -ldflags "-X main.Version=$1" -o /bin/logspout
+apk del go git mercurial build-base
+rm -rf /root/go /var/cache/apk/*
+
+# backwards compatibility
+ln -fs /tmp/docker.sock /var/run/docker.sock
+

logspout/modules.go

@@ -0,0 +1,15 @@
+// https://github.com/gliderlabs/logspout/blob/818dd8260e52d2c148280d86170bdf5267b5c637/modules.go
+
+package main
+
+import (
+	_ "github.com/gliderlabs/logspout/adapters/multiline"
+	_ "github.com/gliderlabs/logspout/adapters/raw"
+	_ "github.com/gliderlabs/logspout/adapters/syslog"
+	_ "github.com/gliderlabs/logspout/healthcheck"
+	_ "github.com/gliderlabs/logspout/httpstream"
+	_ "github.com/gliderlabs/logspout/routesapi"
+	_ "github.com/gliderlabs/logspout/transports/tcp"
+	_ "github.com/gliderlabs/logspout/transports/tls"
+	_ "github.com/gliderlabs/logspout/transports/udp"
+)

logspout/start.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env sh
+
+# start with `logspout` Loggly tag, and add ${audius_loggly_tags} if present
+tag_csv=logspout
+if [[ "${audius_loggly_tags}" ]]; then
+   tag_csv=${tag_csv},${audius_loggly_tags}
+fi
+
+# set hostname to ${audius_discprov_url}, else ${creatorNodeEndpoint}
+if [[ "${audius_discprov_url}" ]]; then
+   hostname=${audius_discprov_url}
+elif [[ "${creatorNodeEndpoint}" ]]; then
+   hostname=${creatorNodeEndpoint}
+fi
+
+# use regex to extract domain in url (source: https://stackoverflow.com/a/2506635/8674706)
+# add extracted domain as a Loggly tag
+if [[ "${hostname}" ]]; then
+   hostname=$(echo ${hostname} | sed -e 's/[^/]*\/\/\([^@]*@\)\?\([^:/]*\).*/\2/')
+   tag_csv=${tag_csv},${hostname}
+fi
+
+# reformat our comma-delimited list
+IFS=","
+for tag in ${tag_csv}
+do
+   tags="${tags} tag=\"${tag}\""
+done
+
+# set and echo our Loggly token and tags for Logspout
+export SYSLOG_STRUCTURED_DATA="$(echo ${audius_loggly_token} | base64 -d)@41058 ${tags}"
+echo SYSLOG_STRUCTURED_DATA=${SYSLOG_STRUCTURED_DATA}
+
+# start logspout and point it to Loggly
+/bin/logspout multiline+syslog+tcp://logs-01.loggly.com:514