Merge pull request #13 from terascope/log-errors-instead-of-exiting

Log errors related to gathering stats and updating metrics instead of exiting the process
terascope · Jan 3, 2024 · 411c203 · 411c203
2 parents 473509b + 1b9ee5b
commit 411c203
Show file tree

Hide file tree

Showing 7 changed files with 1,818 additions and 1,202 deletions.
diff --git a/README.md b/README.md
@@ -85,4 +85,5 @@ update is completed.
 # HELP teraslice_execution_workers Number of workers defined on the execution.  Note that the number of actual workers can differ from this value.
 # HELP teraslice_master_info Information about the Teraslice master node.
 # HELP teraslice_query_duration Total time to complete the named query, in ms.
+# HELP teraslice_exporter_errors Number of errors encountered by teraslice exporter.
 ```
diff --git a/package.json b/package.json
@@ -1,27 +1,27 @@
 {
     "author": "Terascope, LLC <[email protected]>",
-    "version": "0.2.0",
+    "version": "0.3.0",
     "license": "MIT",
     "dependencies": {
         "bunyan": "^1.8.14",
-        "express": "^4.17.1",
+        "express": "^4.18.2",
         "got": "^11.6.2",
-        "prom-client": "^12.0.0"
+        "prom-client": "^15.1.0"
     },
     "devDependencies": {
-        "@terascope/eslint-config": "^0.5.0",
-        "@types/bunyan": "^1.8.6",
-        "@types/express": "^4.17.8",
-        "@types/jest": "^27.4.1",
-        "@types/node": "^14.10.1",
-        "@typescript-eslint/eslint-plugin": "^4.3.0",
-        "@typescript-eslint/parser": "^4.3.0",
-        "eslint": "^7.10.0",
-        "eslint-config-airbnb-base": "^14.2.0",
-        "eslint-plugin-import": "^2.22.1",
-        "jest": "^28.0.1",
-        "ts-jest": "^27.1.4",
-        "typescript": "^4.6.3"
+        "@terascope/eslint-config": "^0.8.0",
+        "@types/bunyan": "^1.8.11",
+        "@types/express": "^4.17.21",
+        "@types/jest": "^29.5.11",
+        "@types/node": "18.16.18",
+        "@typescript-eslint/eslint-plugin": "^6.17.0",
+        "@typescript-eslint/parser": "^6.17.0",
+        "eslint": "8.56.0",
+        "eslint-config-airbnb-base": "^15.0.0",
+        "eslint-plugin-import": "^2.29.1",
+        "jest": "^29.7.0",
+        "ts-jest": "^29.1.1",
+        "typescript": "5.3.3"
     },
     "scripts": {
         "build": "rm -rf dist/ && npx tsc",

diff --git a/src/index.ts b/src/index.ts
@@ -64,29 +64,32 @@ async function main() {
     try {
         await terasliceStats.update();
     } catch (error) {
-        logger.error(`Error encountered getting terasliceStats: ${error}`);
-        process.exit(1);
+        logger.warn(`Error encountered getting terasliceStats: ${error}`);
+        terasliceStats.updateErrors('stats');
     }
     try {
         updateTerasliceMetrics(terasliceStats);
     } catch (error) {
-        logger.error(`Error processing Teraslice cluster state: ${error}`);
-        process.exit(2);
+        logger.warn(`Error processing Teraslice cluster state: ${error}`);
+        terasliceStats.updateErrors('metrics');
     }
 
     setInterval(async () => {
         logger.info('Beginning update of Teraslice state');
         try {
             await terasliceStats.update();
-        } catch (error) {
-            logger.error(`Error encountered getting terasliceStats: ${error}`);
-            process.exit(1)
+                    } catch (error) {
+            logger.warn(`Error encountered getting terasliceStats: ${error}`);
+            // TODO: record stats on the specific endpoint that had an error
+            terasliceStats.updateErrors('stats');
+            return;
         }
         try {
             updateTerasliceMetrics(terasliceStats);
         } catch (error) {
-            logger.error(`Error processing Teraslice cluster state: ${error}`);
-            process.exit(2);
+            logger.warn(`Error processing Teraslice cluster state: ${error}`);
+            terasliceStats.updateErrors('metrics');
+            return;
         }
 
         const datasetSizes = {

diff --git a/src/interfaces.ts b/src/interfaces.ts
@@ -109,5 +109,11 @@ export interface TerasliceStatsInterface {
     info: TerasliceInfo,
     jobs: any[],
     state: TerasliceClusterState,
-    queryDuration: TerasliceQueryDuration
+    queryDuration: TerasliceQueryDuration,
+    errors: TerasliceStatsErrors
+}
+
+export interface TerasliceStatsErrors {
+    statsErrors: number,
+    metricsErrors: number
 }
diff --git a/src/metrics.ts b/src/metrics.ts
@@ -187,6 +187,13 @@ const gaugeExWorkers = new Gauge({
     registers: [metricsRegistry],
 });
 
+const gaugeTerasliceExporterErrors = new Gauge({
+    name: `${metricPrefix}_exporter_errors`,
+    help: 'Number of errors encountered by teraslice exporter.',
+    labelNames: ['error_type'],
+    registers: [metricsRegistry],
+});
+
 /**
  * parseController adds the teraslice execution controller metrics to the
  * metricsRegistry for a single execution.
@@ -426,6 +433,11 @@ function generateExecutionVersions(terasliceStats:TerasliceStats, labels:any) {
     }
 }
 
+function generateExporterErrorStats(terasliceStats:TerasliceStats) {
+        gaugeTerasliceExporterErrors.set({'error_type': 'update_stats_errors' }, terasliceStats.errors.statsErrors);
+        gaugeTerasliceExporterErrors.set({'error_type': 'update_metrics_errors' }, terasliceStats.errors.metricsErrors);
+}
+
 export function updateTerasliceMetrics(terasliceStats: TerasliceStats): void {
     metricsRegistry.resetMetrics();
 
@@ -443,6 +455,7 @@ export function updateTerasliceMetrics(terasliceStats: TerasliceStats): void {
     generateControllerStats(terasliceStats, globalLabels);
     generateExecutionStats(terasliceStats, globalLabels);
     generateExecutionVersions(terasliceStats, globalLabels);
+    generateExporterErrorStats(terasliceStats);
 
     gaugeQueryDuration.set(
         { query_name: 'info', ...globalLabels },

diff --git a/src/teraslice-stats.ts b/src/teraslice-stats.ts
@@ -3,7 +3,8 @@ import {
     TerasliceStatsInterface,
     GetTerasliceApiResponse,
     TerasliceClusterState,
-    TerasliceQueryDuration
+    TerasliceQueryDuration,
+    TerasliceStatsErrors
 } from './interfaces';
 import { pDelay } from './util';
 
@@ -24,6 +25,8 @@ export default class TerasliceStats implements TerasliceStatsInterface {
 
     queryDuration: TerasliceQueryDuration;
 
+    errors: TerasliceStatsErrors;
+
     constructor(baseUrl:string, displayUrl:string) {
         this.baseUrl = new URL(baseUrl);
         this.displayUrl = displayUrl;
@@ -39,6 +42,10 @@ export default class TerasliceStats implements TerasliceStatsInterface {
             jobs: 0,
             state: 0,
         };
+        this.errors = {
+            statsErrors: 0,
+            metricsErrors: 0
+        }
     }
 
     async getTerasliceApi(path:string):Promise<GetTerasliceApiResponse> {
@@ -115,4 +122,13 @@ export default class TerasliceStats implements TerasliceStatsInterface {
         };
         await run();
     }
+
+    updateErrors(errorType: string):void {
+        if (errorType === 'stats') {
+            this.errors.statsErrors++;
+        }
+        else if (errorType === 'metrics') {
+            this.errors.metricsErrors++;
+        }
+    }
 }