Merge branch 'master' into ss-cn-prom

.circleci/config.yml

@@ -224,7 +224,7 @@ jobs:
 
   test-contracts:
     docker:
-      - image: circleci/node:10.16.3
+      - image: circleci/node:16
       - image: trufflesuite/ganache-cli:latest
         command: ['--port=8555', '-a', '100', '-l', '8000000']
     steps:
@@ -265,7 +265,7 @@ jobs:
     # uses medium (2vcpu/4gb) by default
     resource_class: medium
     docker:
-      - image: circleci/node:14.17.3
+      - image: circleci/node:16
       - image: trufflesuite/ganache-cli:latest
         command: ['--port=8546', '-a', '50', '-l', '8000000']
     steps:
@@ -340,8 +340,9 @@ jobs:
             export redisPort=6379
             export spOwnerWallet='yes'
             export isCIBuild=true
-            npm run test:unit
-            npm run test:ci
+            npm run test:coverage:ci
+      - coveralls/upload:
+          path_to_lcov: ./creator-node/coverage/lcov.info
 
   test-discovery-provider:
     docker:
@@ -636,6 +637,9 @@ jobs:
     parameters:
       repo:
         type: string
+      logspout-tag:
+        type: string
+        default: ""
     steps:
       - checkout
       - setup_remote_docker:
@@ -644,10 +648,11 @@ jobs:
           name: Get tag or latest
           command: |
             # order of precendence for IMAGE tag is:
-            # 1. CIRCLE_TAG if defined,
-            # 2. Branch name if 'hotfix' in branch name (branch name passed in via $CIRCLE_BRANCH)
-            # 3. 'latest', which is the default
-            echo "export IMAGE_TAG=`[ $CIRCLE_TAG ] && echo $(echo $CIRCLE_TAG | cut -d@ -f3) || [[ "$CIRCLE_BRANCH" =~ (hotfix) ]] && echo $CIRCLE_BRANCH || echo "latest" `" | tee -a $BASH_ENV
+            # 1. $(head -n1 logging/logspout/Dockerfile) if << parameters.logspout-tag >> defined,
+            # 2. CIRCLE_TAG if defined,
+            # 3. Branch name if 'hotfix' in branch name (branch name passed in via $CIRCLE_BRANCH)
+            # 4. 'latest', which is the default
+            echo "export IMAGE_TAG=`[ << parameters.logspout-tag >> ] && echo $(head -n1 logspout/Dockerfile | cut -f 2 -d ':') || [ $CIRCLE_TAG ] && echo $(echo $CIRCLE_TAG | cut -d@ -f3) || [[ "$CIRCLE_BRANCH" =~ (hotfix) ]] && echo $CIRCLE_BRANCH || echo "latest" `" | tee -a $BASH_ENV
       - run:
           name: Docker login
           command: |
@@ -656,7 +661,8 @@ jobs:
           name: Docker build << parameters.repo >>
           command: |
             cd << parameters.repo >>
-            docker build -t audius/<< parameters.repo >>:$IMAGE_TAG -t audius/<< parameters.repo>>:$(git rev-parse HEAD) --build-arg git_sha=$(git rev-parse HEAD) --build-arg audius_loggly_disable=$audius_loggly_disable --build-arg audius_loggly_token=$audius_loggly_token --build-arg audius_loggly_tags=$audius_loggly_tags --build-arg BUILD_NUM=$CIRCLE_BUILD_NUM .
+            audius_loggly_token_b64=$(echo ${audius_loggly_token} | base64)
+            docker build -t audius/<< parameters.repo >>:$IMAGE_TAG -t audius/<< parameters.repo>>:$(git rev-parse HEAD) --build-arg git_sha=$(git rev-parse HEAD) --build-arg audius_loggly_disable=$audius_loggly_disable --build-arg audius_loggly_token=$audius_loggly_token --build-arg audius_loggly_tags=$audius_loggly_tags --build-arg BUILD_NUM=$CIRCLE_BUILD_NUM --build-arg audius_loggly_token=$audius_loggly_token_b64 .
             docker image prune --filter label=prune=true --filter label=build=$CIRCLE_BUILD_NUM --force
       - run:
           name: Docker push << parameters.repo >>
@@ -732,6 +738,15 @@ workflows:
           name: build-identity-service
           repo: identity-service
 
+      - hold-build-logspout:
+          type: approval
+      - docker-build-and-push:
+          name: build-logspout
+          repo: logspout
+          logspout-tag: "true"
+          requires:
+            - hold-build-logspout
+
       - test-solana-programs:
           name: test-solana-programs
       - test-solana-programs-anchor:

contracts/scripts/call-function.sh

@@ -5,6 +5,6 @@ set -e
 
 # Usage: call-function.sh contractName functionName
 
-node_modules/.bin/truffle console << EOF
+node_modules/.bin/truffle console <<EOF
   $1.deployed().then((instance) => instance.$2.call()).then((value) => console.log("Function returned: " + value))
 EOF

contracts/scripts/circleci-test.sh

@@ -13,16 +13,12 @@ sh ./scripts/lint.sh
 
 printf '\nSTART Truffle tests:\n\n'
 
-if [ $# -eq 0 ]
-	then
-		node_modules/.bin/truffle test --network test_local
-elif [ $1 == '--verbose-rpc' ] && [ $# -eq 1 ]
-	then
-		node_modules/.bin/truffle test --verbose-rpc --network test_local
-elif [ $1 == '--verbose-rpc' ] && [ $# -eq 2 ]
-	then
-		node_modules/.bin/truffle test --verbose-rpc $2 --network test_local
+if [ $# -eq 0 ]; then
+	node_modules/.bin/truffle test --network test_local
+elif [ $1 == '--verbose-rpc' ] && [ $# -eq 1 ]; then
+	node_modules/.bin/truffle test --verbose-rpc --network test_local
+elif [ $1 == '--verbose-rpc' ] && [ $# -eq 2 ]; then
+	node_modules/.bin/truffle test --verbose-rpc $2 --network test_local
 else
 	node_modules/.bin/truffle test $1 --network test_local
 fi
-

contracts/scripts/lint.sh

@@ -17,5 +17,3 @@ printf 'START Solium lint'
 npm run solidity-lint
 printf '\nEND Solium lint'
 printline
-
-

contracts/scripts/truffle-test.sh

@@ -12,7 +12,7 @@ COMMENT
 
 # start ganache container,
 # kill old container instance if tear down did not complete
-if docker ps | grep 'audius_ganache_cli_test' > /dev/null; then
+if docker ps | grep 'audius_ganache_cli_test' >/dev/null; then
 	# killing the container seems to be faster than restarting
 	printf 'Remove old containers and build artifacts\n'
 	docker rm -f audius_ganache_cli_test
@@ -31,15 +31,12 @@ docker run --name audius_ganache_cli_test -d -p 8555:8545 trufflesuite/ganache-c
 ./scripts/lint.sh
 
 # run truffle tests
-if [ $# -eq 0 ]
-	then
-		node_modules/.bin/truffle test --network=test_local
-elif [ $1 == '--verbose-rpc' ] && [ $# -eq 1 ]
-	then
-		node_modules/.bin/truffle test --network=test_local --verbose-rpc
-elif [ $1 == '--verbose-rpc' ] && [ $# -eq 2 ]
-	then
-		node_modules/.bin/truffle test --network=test_local --verbose-rpc $2
+if [ $# -eq 0 ]; then
+	node_modules/.bin/truffle test --network=test_local
+elif [ $1 == '--verbose-rpc' ] && [ $# -eq 1 ]; then
+	node_modules/.bin/truffle test --network=test_local --verbose-rpc
+elif [ $1 == '--verbose-rpc' ] && [ $# -eq 2 ]; then
+	node_modules/.bin/truffle test --network=test_local --verbose-rpc $2
 else
 	node_modules/.bin/truffle test --network=test_local $1
 fi

creator-node/.version.json

@@ -1,4 +1,4 @@
 {
-  "version": "0.3.58",
+  "version": "0.3.59",
   "service": "content-node"
 }

creator-node/default-config.json

@@ -31,7 +31,7 @@
   "ethWallets": "",
   "spOwnerWalletIndex": 1,
   "spOwnerWallet": "",
-  "discoveryProviderWhitelist": "http://dn1_web-server_1:5000",
+  "discoveryProviderWhitelist": "",
   "identityService": "http://audius-identity-service_identity-service_1:7000",
   "dataRegistryAddress": "",
   "dataProviderUrl": "http://docker.for.mac.localhost:8545",

creator-node/package.json

@@ -10,8 +10,8 @@
     "test:local:teardown": "./scripts/run-tests.sh standalone_creator teardown",
     "test:teardown": "./scripts/run-tests.sh teardown",
     "test:unit": "./scripts/run-tests.sh unit_test",
-    "coverage": "nyc npm run test",
-    "report": "nyc report --reporter=html",
+    "test:coverage": "nyc --reporter=lcov --reporter=text npm run test",
+    "test:coverage:ci": "nyc --reporter=lcov npm run test:ci && nyc report --reporter=text-lcov | coveralls",
     "lint:fix": "eslint --fix --ext=js,ts src",
     "lint": "eslint --ext=js,ts src"
   },
@@ -75,6 +75,7 @@
     "@typescript-eslint/parser": "^5.5.0",
     "chai": "4.3.6",
     "chai-as-promised": "^7.1.1",
+    "coveralls": "3.1.1",
     "eslint": "^7.32.0",
     "eslint-config-standard": "^16.0.3",
     "eslint-plugin-import": "^2.25.3",
@@ -84,7 +85,7 @@
     "mocha": "^5.2.0",
     "nock": "^13.1.0",
     "nodemon": "^1.19.4",
-    "nyc": "^15.0.0",
+    "nyc": "15.1.0",
     "prettier": "^2.5.1",
     "proxyquire": "^2.1.3",
     "sequelize-cli": "^5.3.0",

creator-node/scripts/start.sh

@@ -7,13 +7,18 @@ if [[ "$WAIT_HOSTS" != "" ]]; then
     /usr/bin/wait
 fi
 
-if [[ -z "$logglyDisable" ]]; then
-    if [[ -n "$logglyToken" ]]; then
-        logglyTags=$(echo $logglyTags | python3 -c "print(' '.join(f'tag=\\\\\"{i}\\\\\"' for i in input().split(',')))")
-        mkdir -p /var/spool/rsyslog
-        mkdir -p /etc/rsyslog.d
-        sed -i '1s|^|$MaxMessageSize 64k\n|' /etc/rsyslog.conf
-        cat >/etc/rsyslog.d/22-loggly.conf <<EOF
+# enable rsyslog if not explicitly disabled by audius-docker-compose
+: "${enableRsyslog:=true}"
+
+# $enableRsyslog should be true
+# $logglyDisable should be empty/null
+# $logglyToken should be a nonzero length string
+if $enableRsyslog && [[ -z "$logglyDisable" && -n "$logglyToken" ]]; then
+    logglyTags=$(echo $logglyTags | python3 -c "print(' '.join(f'tag=\\\\\"{i}\\\\\"' for i in input().split(',')))")
+    mkdir -p /var/spool/rsyslog
+    mkdir -p /etc/rsyslog.d
+    sed -i '1s|^|$MaxMessageSize 64k\n|' /etc/rsyslog.conf
+    cat >/etc/rsyslog.d/22-loggly.conf <<EOF
 \$WorkDirectory /var/spool/rsyslog # where to place spool files
 \$ActionQueueFileName fwdRule1   # unique name prefix for spool files
 \$ActionQueueMaxDiskSpace 1g    # 1gb space limit (use as much as possible)
@@ -25,8 +30,7 @@ template(name="LogglyFormat" type="string"
 # Send messages to Loggly over TCP using the template.
 action(type="omfwd" protocol="tcp" target="logs-01.loggly.com" port="514" template="LogglyFormat")
 EOF
-        rsyslogd
-    fi
+    rsyslogd
 fi
 
 if [ -z "$redisHost" ]; then

creator-node/src/apiHelpers.js

@@ -3,7 +3,8 @@ const config = require('./config')
 const {
   requestNotExcludedFromLogging,
   getDuration,
-  createChildLogger
+  createChildLogger,
+  logger: genericLogger
 } = require('./logging')
 const { generateTimestampAndSignature } = require('./apiSigning')
 
@@ -29,7 +30,7 @@ module.exports.handleResponse = (func) => {
 
       next()
     } catch (error) {
-      console.error('HandleResponse', error)
+      genericLogger.error('HandleResponse', error)
       next(error)
     }
   }

creator-node/src/blacklistManager.js

@@ -719,12 +719,12 @@ class BlacklistManager {
       }
     })
     stream.on('end', function () {
-      console.log(
+      logger.info(
         `Done deleting ${REDIS_MAP_BLACKLIST_SEGMENTCID_TO_TRACKID_KEY} entries`
       )
     })
     stream.on('error', function (e) {
-      console.error(
+      logger.error(
         `Could not delete ${REDIS_MAP_BLACKLIST_SEGMENTCID_TO_TRACKID_KEY} entries: ${e.toString()}`
       )
     })

creator-node/src/redis.js

@@ -1,29 +1,30 @@
 const config = require('./config.js')
+const { logger } = require('./logging')
 const Redis = require('ioredis')
 
 const redisClient = new Redis(config.get('redisPort'), config.get('redisHost'))
 
 const EXPIRATION = 60 * 60 * 2 // 2 hours in seconds
 class RedisLock {
   static async setLock(key, expiration = EXPIRATION) {
-    console.log(`SETTING LOCK ${key}`)
+    logger.info(`SETTING LOCK ${key}`)
     // set allows you to set an optional expire param
     return redisClient.set(key, true, 'EX', expiration)
   }
 
   static async getLock(key) {
-    console.log(`GETTING LOCK ${key}`)
+    logger.info(`GETTING LOCK ${key}`)
     return redisClient.get(key)
   }
 
   static async acquireLock(key, expiration = EXPIRATION) {
-    console.log(`SETTING LOCK IF NOT EXISTS ${key}`)
+    logger.info(`SETTING LOCK IF NOT EXISTS ${key}`)
     const response = await redisClient.set(key, true, 'NX', 'EX', expiration)
     return !!response
   }
 
   static async removeLock(key) {
-    console.log(`DELETING LOCK ${key}`)
+    logger.info(`DELETING LOCK ${key}`)
     return redisClient.del(key)
   }
 }

creator-node/src/reqLimiter.js

@@ -1,6 +1,7 @@
 const express = require('express')
 const rateLimit = require('express-rate-limit')
 const config = require('./config.js')
+const { logger } = require('./logging')
 const RedisStore = require('rate-limit-redis')
 const client = require('./redis.js')
 const { verifyRequesterIsValidSP } = require('./apiSigning.js')
@@ -9,7 +10,7 @@ let endpointRateLimits = {}
 try {
   endpointRateLimits = JSON.parse(config.get('endpointRateLimits'))
 } catch (e) {
-  console.error('Failed to parse endpointRateLimits!')
+  logger.error('Failed to parse endpointRateLimits!')
 }
 
 // Key generator for rate limiter that rate limits based on unique IP

creator-node/src/routes/files.js

@@ -133,7 +133,7 @@ const logGetCIDDecisionTree = (decisionTree, req) => {
   try {
     req.logger.info(`[getCID] Decision Tree: ${JSON.stringify(decisionTree)}`)
   } catch (e) {
-    console.error(`[getCID] Decision Tree - Failed to print: ${e.message}`)
+    req.logger.error(`[getCID] Decision Tree - Failed to print: ${e.message}`)
   }
 }
 

creator-node/src/services/TrustedNotifierManager.js

@@ -64,7 +64,7 @@ class TrustedNotifierManager {
 
       this.trustedNotifierData = this.trustedNotifierChainData
     } catch (e) {
-      console.error(e)
+      this.logError(`Failed to initialize: ${e}`)
     }
   }