CON-181 Add prometheus metrics to CN

creator-node/package.json

@@ -58,6 +58,7 @@
     "multer": "^1.4.0",
     "pg": "^8.0.3",
     "prettier-config-standard": "^4.0.0",
+    "prom-client": "^14.0.1",
     "promise.any": "^2.0.2",
     "rate-limit-redis": "^1.7.0",
     "sequelize": "^4.44.4",

creator-node/src/components/healthCheck/healthCheckController.js

@@ -131,8 +131,22 @@ const healthCheckController = async (req) => {
  * Controller for `health_check/sync` route, calls
  * syncHealthCheckController
  */
-const syncHealthCheckController = async () => {
+const syncHealthCheckController = async (req) => {
   const response = await syncHealthCheck(serviceRegistry)
+
+  const prometheusRegistry = req.app.get('serviceRegistry').prometheusRegistry
+  const syncQueueJobsTotalMetric = prometheusRegistry.getMetric(
+    prometheusRegistry.metricNames.SYNC_QUEUE_JOBS_TOTAL_GAUGE
+  )
+  syncQueueJobsTotalMetric.set(
+    { status: 'manual_waiting' },
+    response.manualWaitingCount
+  )
+  syncQueueJobsTotalMetric.set(
+    { status: 'recurring_waiting' },
+    response.recurringWaitingCount
+  )
+
   return successResponse(response)
 }
 

creator-node/src/monitors/filesystem.js

@@ -1,5 +1,6 @@
 const si = require('systeminformation')
 const disk = require('diskusage')
+
 const DiskManager = require('../diskManager')
 
 const getStoragePathSize = async () => {
@@ -11,7 +12,8 @@ const getStoragePathSize = async () => {
 const getStoragePathUsed = async () => {
   const storagePath = DiskManager.getConfigStoragePath()
   const { available, total } = await disk.check(storagePath)
-  return total - available
+  const used = total - available
+  return used
 }
 
 // We first check '/var/k8s' in case the service operator has elected to

creator-node/src/routes/prometheusMetricsRoutes.js

@@ -0,0 +1,12 @@
+/**
+ * Exposes Prometheus metrics at `GET /prometheus_metrics`
+ */
+module.exports = function (app) {
+  app.get('/prometheus_metrics', async (req, res) => {
+    const prometheusRegistry = req.app.get('serviceRegistry').prometheusRegistry
+    const metricData = await prometheusRegistry.getAllMetricData()
+
+    res.setHeader('Content-Type', prometheusRegistry.registry.contentType)
+    return res.end(metricData)
+  })
+}

creator-node/src/routes/tracks.js

@@ -304,8 +304,17 @@ module.exports = function (app) {
         transcodedTrackUUID
       } = req.body
 
+      const prometheusRegistry =
+        req.app.get('serviceRegistry').prometheusRegistry
+      const routePostTracksDurationSecondsMetric = prometheusRegistry.getMetric(
+        prometheusRegistry.metricNames
+          .ROUTE_POST_TRACKS_DURATION_SECONDS_HISTOGRAM
+      )
+      const metricEndTimerFn = routePostTracksDurationSecondsMetric.startTimer()
+
       // Input validation
       if (!blockchainTrackId || !blockNumber || !metadataFileUUID) {
+        metricEndTimerFn({ code: 400 })
         return errorResponseBadRequest(
           'Must include blockchainTrackId, blockNumber, and metadataFileUUID.'
         )
@@ -314,6 +323,7 @@ module.exports = function (app) {
       // Error on outdated blocknumber
       const cnodeUser = req.session.cnodeUser
       if (blockNumber < cnodeUser.latestBlockNumber) {
+        metricEndTimerFn({ code: 400 })
         return errorResponseBadRequest(
           `Invalid blockNumber param ${blockNumber}. Must be greater or equal to previously processed blocknumber ${cnodeUser.latestBlockNumber}.`
         )
@@ -325,6 +335,7 @@ module.exports = function (app) {
         where: { fileUUID: metadataFileUUID, cnodeUserUUID }
       })
       if (!file) {
+        metricEndTimerFn({ code: 400 })
         return errorResponseBadRequest(
           `No file db record found for provided metadataFileUUID ${metadataFileUUID}.`
         )
@@ -339,11 +350,13 @@ module.exports = function (app) {
           !Array.isArray(metadataJSON.track_segments) ||
           !metadataJSON.track_segments.length
         ) {
+          metricEndTimerFn({ code: 500 })
           return errorResponseServerError(
             `Malformatted metadataJSON stored for metadataFileUUID ${metadataFileUUID}.`
           )
         }
       } catch (e) {
+        metricEndTimerFn({ code: 500 })
         return errorResponseServerError(
           `No file stored on disk for metadataFileUUID ${metadataFileUUID} at storagePath ${file.storagePath}.`
         )
@@ -357,6 +370,7 @@ module.exports = function (app) {
           metadataJSON.cover_art_sizes
         )
       } catch (e) {
+        metricEndTimerFn({ code: 500 })
         return errorResponseServerError(e.message)
       }
 
@@ -543,10 +557,12 @@ module.exports = function (app) {
 
         await issueAndWaitForSecondarySyncRequests(req)
 
+        metricEndTimerFn({ code: 200 })
         return successResponse()
       } catch (e) {
         req.logger.error(e.message)
         await transaction.rollback()
+        metricEndTimerFn({ code: 500 })
         return errorResponseServerError(e.message)
       }
     })

creator-node/src/serviceRegistry.js

@@ -1,3 +1,7 @@
+const { createBullBoard } = require('@bull-board/api')
+const { BullAdapter } = require('@bull-board/api/bullAdapter')
+const { ExpressAdapter } = require('@bull-board/express')
+
 const { libs: AudiusLibs } = require('@audius/sdk')
 const redisClient = require('./redis')
 const BlacklistManager = require('./blacklistManager')
@@ -6,10 +10,6 @@ const config = require('./config')
 const URSMRegistrationManager = require('./services/URSMRegistrationManager')
 const { logger } = require('./logging')
 const utils = require('./utils')
-const { createBullBoard } = require('@bull-board/api')
-const { BullAdapter } = require('@bull-board/api/bullAdapter')
-const { ExpressAdapter } = require('@bull-board/express')
-
 const MonitoringQueue = require('./monitors/MonitoringQueue')
 const SyncQueue = require('./services/sync/syncQueue')
 const SkippedCIDsRetryQueue = require('./services/sync/skippedCIDsRetryService')
@@ -19,6 +19,7 @@ const TrustedNotifierManager = require('./services/TrustedNotifierManager')
 const ImageProcessingQueue = require('./ImageProcessingQueue')
 const TranscodingQueue = require('./TranscodingQueue')
 const StateMachineManager = require('./services/stateMachineManager')
+const PrometheusRegistry = require('./services/prometheusMonitoring/prometheusRegistry')
 
 /**
  * `ServiceRegistry` is a container responsible for exposing various
@@ -45,6 +46,7 @@ class ServiceRegistry {
     this.blacklistManager = BlacklistManager
     this.monitoringQueue = new MonitoringQueue()
     this.sessionExpirationQueue = new SessionExpirationQueue()
+    this.prometheusRegistry = new PrometheusRegistry()
 
     // below services are initialized separately in below functions `initServices()` and `initServicesThatRequireServer()`
     this.libs = null