CON-181 Add prometheus metrics to CN

creator-node/package.json

@@ -58,6 +58,7 @@
     "multer": "^1.4.0",
     "pg": "^8.0.3",
     "prettier-config-standard": "^4.0.0",
+    "prom-client": "^14.0.1",
     "promise.any": "^2.0.2",
     "rate-limit-redis": "^1.7.0",
     "sequelize": "^4.44.4",

creator-node/src/apiHelpers.js

@@ -18,6 +18,7 @@ module.exports.handleResponse = (func) => {
       }
 
       sendResponse(req, res, resp)
+
       next()
     } catch (error) {
       genericLogger.error('HandleResponse', error)

creator-node/src/monitors/filesystem.js

@@ -1,17 +1,34 @@
 const si = require('systeminformation')
 const disk = require('diskusage')
+
 const DiskManager = require('../diskManager')
+const serviceRegistry = require('../serviceRegistry')
+
+const prometheusRegistry = serviceRegistry.prometheusRegistry
 
 const getStoragePathSize = async () => {
   const storagePath = DiskManager.getConfigStoragePath()
   const { total } = await disk.check(storagePath)
+
+  const storagePathSizeGaugeMetric = prometheusRegistry.getMetric(
+    prometheusRegistry.metricNames.STORAGE_PATH_SIZE_GAUGE
+  )
+  storagePathSizeGaugeMetric.set(total)
+
   return total
 }
 
 const getStoragePathUsed = async () => {
   const storagePath = DiskManager.getConfigStoragePath()
   const { available, total } = await disk.check(storagePath)
-  return total - available
+  const used = total - available
+
+  const storagePathUsedGaugeMetric = prometheusRegistry.getMetric(
+    prometheusRegistry.metricNames.STORAGE_PATH_USED_GAUGE
+  )
+  storagePathUsedGaugeMetric.set(total)
+
+  return used
 }
 
 // We first check '/var/k8s' in case the service operator has elected to

creator-node/src/routes/prometheusMetricsRoutes.js

@@ -0,0 +1,9 @@
+module.exports = function (app) {
+  app.get('/prometheus_metrics', async (req, res) => {
+    const prometheusRegistry = req.app.get('serviceRegistry').prometheusRegistry
+    const metricData = await prometheusRegistry.getAllMetricData()
+
+    res.setHeader('Content-Type', prometheusRegistry.registry.contentType)
+    return res.end(metricData)
+  })
+}

creator-node/src/routes/tracks.js

@@ -304,8 +304,17 @@ module.exports = function (app) {
         transcodedTrackUUID
       } = req.body
 
+      const prometheusRegistry =
+        req.app.get('serviceRegistry').prometheusRegistry
+      const routePostTracksDurationSecondsMetric = prometheusRegistry.getMetric(
+        prometheusRegistry.metricNames
+          .ROUTE_POST_TRACKS_DURATION_SECONDS_HISTOGRAM
+      )
+      const metricEndTimerFn = routePostTracksDurationSecondsMetric.startTimer()
+
       // Input validation
       if (!blockchainTrackId || !blockNumber || !metadataFileUUID) {
+        metricEndTimerFn({ code: 400 })
         return errorResponseBadRequest(
           'Must include blockchainTrackId, blockNumber, and metadataFileUUID.'
         )
@@ -314,6 +323,7 @@ module.exports = function (app) {
       // Error on outdated blocknumber
       const cnodeUser = req.session.cnodeUser
       if (blockNumber < cnodeUser.latestBlockNumber) {
+        metricEndTimerFn({ code: 400 })
         return errorResponseBadRequest(
           `Invalid blockNumber param ${blockNumber}. Must be greater or equal to previously processed blocknumber ${cnodeUser.latestBlockNumber}.`
         )
@@ -325,6 +335,7 @@ module.exports = function (app) {
         where: { fileUUID: metadataFileUUID, cnodeUserUUID }
       })
       if (!file) {
+        metricEndTimerFn({ code: 400 })
         return errorResponseBadRequest(
           `No file db record found for provided metadataFileUUID ${metadataFileUUID}.`
         )
@@ -339,11 +350,13 @@ module.exports = function (app) {
           !Array.isArray(metadataJSON.track_segments) ||
           !metadataJSON.track_segments.length
         ) {
+          metricEndTimerFn({ code: 500 })
           return errorResponseServerError(
             `Malformatted metadataJSON stored for metadataFileUUID ${metadataFileUUID}.`
           )
         }
       } catch (e) {
+        metricEndTimerFn({ code: 500 })
         return errorResponseServerError(
           `No file stored on disk for metadataFileUUID ${metadataFileUUID} at storagePath ${file.storagePath}.`
         )
@@ -357,6 +370,7 @@ module.exports = function (app) {
           metadataJSON.cover_art_sizes
         )
       } catch (e) {
+        metricEndTimerFn({ code: 500 })
         return errorResponseServerError(e.message)
       }
 
@@ -543,10 +557,12 @@ module.exports = function (app) {
 
         await issueAndWaitForSecondarySyncRequests(req)
 
+        metricEndTimerFn({ code: 200 })
         return successResponse()
       } catch (e) {
         req.logger.error(e.message)
         await transaction.rollback()
+        metricEndTimerFn({ code: 500 })
         return errorResponseServerError(e.message)
       }
     })

creator-node/src/serviceRegistry.js

@@ -1,3 +1,7 @@
+const { createBullBoard } = require('@bull-board/api')
+const { BullAdapter } = require('@bull-board/api/bullAdapter')
+const { ExpressAdapter } = require('@bull-board/express')
+
 const { libs: AudiusLibs } = require('@audius/sdk')
 const redisClient = require('./redis')
 const BlacklistManager = require('./blacklistManager')
@@ -6,10 +10,6 @@ const config = require('./config')
 const URSMRegistrationManager = require('./services/URSMRegistrationManager')
 const { logger } = require('./logging')
 const utils = require('./utils')
-const { createBullBoard } = require('@bull-board/api')
-const { BullAdapter } = require('@bull-board/api/bullAdapter')
-const { ExpressAdapter } = require('@bull-board/express')
-
 const MonitoringQueue = require('./monitors/MonitoringQueue')
 const SyncQueue = require('./services/sync/syncQueue')
 const SkippedCIDsRetryQueue = require('./services/sync/skippedCIDsRetryService')
@@ -19,6 +19,7 @@ const TrustedNotifierManager = require('./services/TrustedNotifierManager')
 const ImageProcessingQueue = require('./ImageProcessingQueue')
 const TranscodingQueue = require('./TranscodingQueue')
 const StateMachineManager = require('./services/stateMachineManager')
+const PrometheusRegistry = require('./services/prometheusMonitoring/prometheusRegistry')
 
 /**
  * `ServiceRegistry` is a container responsible for exposing various
@@ -54,6 +55,7 @@ class ServiceRegistry {
     this.syncQueue = null
     this.skippedCIDsRetryQueue = null
     this.trustedNotifierManager = null
+    this.prometheusRegistry = null
 
     this.servicesInitialized = false
     this.servicesThatRequireServerInitialized = false
@@ -142,6 +144,8 @@ class ServiceRegistry {
    *  - create bull queue monitoring dashboard, which needs other server-dependent services to be running
    */
   async initServicesThatRequireServer(app) {
+    this.prometheusRegistry = new PrometheusRegistry()
+
     // Cannot progress without recovering spID from node's record on L1 ServiceProviderFactory contract
     // Retries indefinitely
     await this._recoverNodeL1Identity()

creator-node/src/services/prometheusMonitoring/prometheus.constants.js

@@ -0,0 +1,54 @@
+const promClient = require('prom-client')
+
+const METRIC_PREFIX = 'cn_'
+
+/**
+ * @notice Counter and Summary metric types are currently disabled, since we should almost never be using them.
+ *  - Gauge should always be used over Counter since it can be decreased, with no performance overhead.
+ *  - Histogram should always be used over Summary, per docs below
+ * https://prometheus.io/docs/tutorials/understanding_metric_types/
+ */
+const MetricTypes = Object.freeze({
+  GAUGE: promClient.Gauge,
+  HISTOGRAM: promClient.Histogram
+  // COUNTER: promClient.Counter,
+  // SUMMARY: promClient.Summary
+})
+
+const MetricNames = Object.freeze({
+  STORAGE_PATH_SIZE_GAUGE: 'storage_path_size_bytes',
+  STORAGE_PATH_USED_GAUGE: 'storage_path_used_bytes',
+  ROUTE_POST_TRACKS_DURATION_SECONDS_HISTOGRAM:
+    'route_post_tracks_duration_seconds'
+})
+
+const Metrics = Object.freeze({
+  [MetricNames.STORAGE_PATH_SIZE_GAUGE]: {
+    metricType: MetricTypes.GAUGE,
+    metricConfig: {
+      name: MetricNames.STORAGE_PATH_SIZE_GAUGE,
+      help: 'Total disk space (free + used) (bytes)'
+    }
+  },
+  [MetricNames.STORAGE_PATH_USED_GAUGE]: {
+    metricType: MetricTypes.GAUGE,
+    metricConfig: {
+      name: MetricNames.STORAGE_PATH_USED_GAUGE,
+      help: 'Used disk space (bytes)'
+    }
+  },
+  [MetricNames.ROUTE_POST_TRACKS_DURATION_SECONDS_HISTOGRAM]: {
+    metricType: MetricTypes.HISTOGRAM,
+    metricConfig: {
+      name: MetricNames.ROUTE_POST_TRACKS_DURATION_SECONDS_HISTOGRAM,
+      help: 'Duration for POST /tracks route (seconds)',
+      labelNames: ['code'],
+      buckets: [0.1, 0.3, 0.5, 1, 3, 5, 10] // 0.1 to 10 seconds
+    }
+  }
+})
+
+module.exports.METRIC_PREFIX = METRIC_PREFIX
+module.exports.MetricTypes = MetricTypes
+module.exports.MetricNames = MetricNames
+module.exports.Metrics = Metrics