From 1574226147ab96e4c9926a8e8a81d9cf7544cb46 Mon Sep 17 00:00:00 2001 From: Peter Luitjens <43619525+busma13@users.noreply.github.com> Date: Thu, 9 May 2024 14:54:13 -0700 Subject: [PATCH] Refactor PromMetrics to use Collect functions (#3607) This PR makes the following changes: - Refactor `addMetric` into individual functions for counter, gauge, and histogram. - Add `collect()` parameter to all `add` functions. - Update types: - import `prom-client` types into `types` package - remove duplicate types from `terafoundation` and `job-components` and import from `types`. This required some refactoring to not use context as a parameter, as the `context` type differs between `terafoundation` and `job-components`. - Move `info` metrics of slice, worker, and master into the `setPromMetrics()` functions. - Update PromClient docs and all relevant tests --- docs/development/k8s.md | 56 +- e2e/package.json | 4 +- package.json | 2 +- packages/data-mate/package.json | 10 +- packages/data-types/package.json | 6 +- packages/elasticsearch-api/package.json | 8 +- packages/elasticsearch-store/package.json | 12 +- packages/generator-teraslice/package.json | 4 +- packages/job-components/package.json | 4 +- .../src/execution-context/slicer.ts | 57 -- .../src/execution-context/worker.ts | 57 -- .../job-components/src/interfaces/context.ts | 35 +- packages/job-components/src/test-helpers.ts | 141 ++-- .../job-components/test/test-helpers-spec.ts | 20 +- packages/scripts/package.json | 4 +- packages/terafoundation/package.json | 8 +- packages/terafoundation/src/api/index.ts | 7 +- .../src/api/prom-metrics/exporter.ts | 4 +- .../src/api/prom-metrics/prom-metrics-api.ts | 228 +++--- packages/terafoundation/src/interfaces.ts | 44 +- packages/terafoundation/src/test-context.ts | 2 +- packages/terafoundation/test/apis/api-spec.ts | 19 +- .../terafoundation/test/apis/exporter-spec.ts | 4 +- .../test/apis/prom-metrics-spec.ts | 657 ++++++++++-------- .../terafoundation/test/test-context-spec.ts | 4 +- packages/teraslice-cli/package.json | 8 +- packages/teraslice-client-js/package.json | 6 +- packages/teraslice-messaging/package.json | 6 +- .../teraslice-op-test-harness/package.json | 4 +- packages/teraslice-state-storage/package.json | 6 +- packages/teraslice-test-harness/package.json | 4 +- packages/teraslice/package.json | 14 +- .../src/lib/cluster/cluster_master.ts | 116 ++-- .../lib/workers/execution-controller/index.ts | 69 ++ .../teraslice/src/lib/workers/worker/index.ts | 73 ++ packages/ts-transforms/package.json | 8 +- packages/types/package.json | 2 +- packages/types/src/terafoundation.ts | 42 +- packages/utils/package.json | 4 +- packages/xlucene-parser/package.json | 6 +- packages/xlucene-translator/package.json | 8 +- packages/xpressions/package.json | 6 +- 42 files changed, 968 insertions(+), 811 deletions(-) diff --git a/docs/development/k8s.md b/docs/development/k8s.md index 840fa19ca87..da611b77c24 100644 --- a/docs/development/k8s.md +++ b/docs/development/k8s.md @@ -349,17 +349,39 @@ yarn run ts-scripts k8s-env --rebuild --skip-build --reset-store ## Prometheus Metrics API -The `PromMetrics` class lives within `packages/terafoundation/src/api/prom-metrics` package. Use of its API can be enabled using `prom_metrics_enabled` in the terafoundation config and overwritten in the job config. The `init` function can be found at `context.apis.foundation.promMetrics.init`. It is called on startup of the Teraslice master, execution_Controller, and worker, but only creates the API if `prom_metrics_enabled` is true. +The `PromMetrics` class lives within `packages/terafoundation/src/api/prom-metrics` package. Use of its API can be enabled using `prom_metrics_enabled` in the terafoundation config and overwritten in the job config. The `init` function can be found at `context.apis.foundation.promMetrics.init`. It is called on startup of the Teraslice master, execution_controller, and worker, but only creates the API if `prom_metrics_enabled` is true. + +### Functions + + +| Name | Description | Type | +| ---------------- | --------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------- | +| init | initialize the API and create exporter server | (config: PromMetricsInitConfig) => Promise | +| set | set the value of a gauge | (name: string, labels: Record, value: number) => void | +| inc | increment the value of a counter or gauge | (name: string, labelValues: Record, value: number) => void | +| dec | decrement the value of a gauge | (name: string, labelValues: Record, value: number) => void | +| observe | observe a histogram or summary | (name: string, labelValues: Record, value: number) => void | +| addGauge | add a gauge metric | (name: string, help: string, labelNames: Array, collectFn?: CollectFunction) => Promise | +| addCounter | add a counter metric | (name: string, help: string, labelNames: Array, collectFn?: CollectFunction) => Promise | +| addHistogram | add a histogram metric | (name: string, help: string, labelNames: Array, collectFn?: CollectFunction, buckets?: Array) => Promise | +| addSummary | add a summary metric | (name: string, help: string, labelNames: Array, collectFn?: CollectFunction, maxAgeSeconds?: number, ageBuckets?: number, percentiles?: Array) => Promise | +| hasMetric | check if a metric exists | (name: string) => boolean | +| deleteMetric | delete a metric from the metric list | (name: string) => Promise | +| verifyAPI | verfiy that the API is running | () => boolean | +| shutdown | disable API and shutdown exporter server | () => Promise | +| getDefaultLabels | retrieve the default labels set at init | () => Record | Example init: ```typescript await config.context.apis.foundation.promMetrics.init({ - context: config.context, - logger: this.logger, - metrics_enabled_by_job: config.executionConfig.prom_metrics_enabled, // optional job override assignment: 'execution_controller', - port: config.executionConfig.prom_metrics_port, // optional job override - default_metrics: config.executionConfig.prom_metrics_add_default, // optional job override + logger: this.logger, + tf_prom_metrics_add_default: terafoundation.prom_metrics_add_default, + tf_prom_metrics_enabled: terafoundation.prom_metrics_enabled, + tf_prom_metrics_port: terafoundation.prom_metrics_port, + job_prom_metrics_add_default: config.executionConfig.prom_metrics_add_default, // optional job override + job_prom_metrics_enabled: config.executionConfig.prom_metrics_enabled, // optional job override + job_prom_metrics_port: config.executionConfig.prom_metrics_port, // optional job override labels: { // optional default labels on all metrics for this teraslice process ex_id: this.exId, job_id: this.jobId, @@ -370,13 +392,12 @@ await config.context.apis.foundation.promMetrics.init({ Once initialized all of the other functions under `context.apis.foundation.promMetrics` will be enabled. It's important to note that the foundation level wrapper functions allow all of the prom metrics functions to be called even if metrics are disabled or the API hasn't been initialized. There is no need to make checks at the level where a function is called, and failures will never throw errors. -Example addMetric: +Example Counter: ```typescript -await this.context.apis.foundation.promMetrics.addMetric( +await this.context.apis.foundation.promMetrics.addCounter( 'slices_dispatched', // name 'number of slices a slicer has dispatched', // help or description ['class'], // label names specific to this metric - 'counter'); // metric type // now we can increment the counter anywhere else in the code this.context.apis.foundation.promMetrics.inc( @@ -386,6 +407,23 @@ this.context.apis.foundation.promMetrics.inc( ); ``` +Example Gauge using collect() callback: +```typescript +const self = this; // rename `this` to use inside collect() +await this.context.apis.foundation.promMetrics.addGauge( + 'slices_dispatched', // name + 'number of slices a slicer has dispatched', // help or description + ['class'], // label names specific to this metric + function collect() { // callback fn updates value only when '/metrics' endpoint is hit + const slicesFinished = self.getSlicesDispatched(); // get current value from local momory + const labels = { // 'set()' needs both default labels and labels specific to metric to match the correct gauge + ...self.context.apis.foundation.promMetrics.getDefaultLabels(), + class: 'SlicerExecutionContext' + }; + this.set(labels, slicesFinished); // this refers to the Gauge + } +``` + The label names as well as the metric name must match when using `inc`, `dec`, `set`, or `observe` to modify a metric. ## Extras diff --git a/e2e/package.json b/e2e/package.json index 4dd7dacfeb4..2906c963dc1 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -42,9 +42,9 @@ "ms": "^2.1.3" }, "devDependencies": { - "@terascope/types": "^0.16.0", + "@terascope/types": "^0.17.0", "bunyan": "^1.8.15", - "elasticsearch-store": "^0.83.1", + "elasticsearch-store": "^0.84.0", "fs-extra": "^11.2.0", "ms": "^2.1.3", "nanoid": "^3.3.4", diff --git a/package.json b/package.json index 8dce20a01a4..8cc059fc037 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "teraslice-workspace", "displayName": "Teraslice", - "version": "1.4.1", + "version": "1.5.0", "private": true, "homepage": "https://github.com/terascope/teraslice", "bugs": { diff --git a/packages/data-mate/package.json b/packages/data-mate/package.json index 920e0530a61..93f8677b8c3 100644 --- a/packages/data-mate/package.json +++ b/packages/data-mate/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/data-mate", "displayName": "Data-Mate", - "version": "0.55.1", + "version": "0.56.0", "description": "Library of data validations/transformations", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/data-mate#readme", "repository": { @@ -29,9 +29,9 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/data-types": "^0.49.1", - "@terascope/types": "^0.16.0", - "@terascope/utils": "^0.58.1", + "@terascope/data-types": "^0.50.0", + "@terascope/types": "^0.17.0", + "@terascope/utils": "^0.59.0", "@types/validator": "^13.11.9", "awesome-phonenumber": "^2.70.0", "date-fns": "^2.30.0", @@ -46,7 +46,7 @@ "uuid": "^9.0.1", "valid-url": "^1.0.9", "validator": "^13.11.0", - "xlucene-parser": "^0.57.1" + "xlucene-parser": "^0.58.0" }, "devDependencies": { "@types/ip6addr": "^0.2.6", diff --git a/packages/data-types/package.json b/packages/data-types/package.json index 9664c8cbf58..05f5633c936 100644 --- a/packages/data-types/package.json +++ b/packages/data-types/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/data-types", "displayName": "Data Types", - "version": "0.49.1", + "version": "0.50.0", "description": "A library for defining the data structures and mapping", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/data-types#readme", "bugs": { @@ -26,8 +26,8 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/types": "^0.16.0", - "@terascope/utils": "^0.58.1", + "@terascope/types": "^0.17.0", + "@terascope/utils": "^0.59.0", "graphql": "^14.7.0", "lodash": "^4.17.21", "yargs": "^17.7.2" diff --git a/packages/elasticsearch-api/package.json b/packages/elasticsearch-api/package.json index 11bf2e9dc48..2500693a7c7 100644 --- a/packages/elasticsearch-api/package.json +++ b/packages/elasticsearch-api/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/elasticsearch-api", "displayName": "Elasticsearch API", - "version": "3.19.1", + "version": "3.20.0", "description": "Elasticsearch client api used across multiple services, handles retries and exponential backoff", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-api#readme", "bugs": { @@ -23,8 +23,8 @@ "test:watch": "TEST_RESTRAINED_ELASTICSEARCH='true' ts-scripts test --watch . --" }, "dependencies": { - "@terascope/types": "^0.16.0", - "@terascope/utils": "^0.58.1", + "@terascope/types": "^0.17.0", + "@terascope/utils": "^0.59.0", "bluebird": "^3.7.2", "setimmediate": "^1.0.5" }, @@ -32,7 +32,7 @@ "@opensearch-project/opensearch": "^1.2.0", "@types/elasticsearch": "^5.0.43", "elasticsearch": "^15.4.1", - "elasticsearch-store": "^0.83.1", + "elasticsearch-store": "^0.84.0", "elasticsearch6": "npm:@elastic/elasticsearch@^6.7.0", "elasticsearch7": "npm:@elastic/elasticsearch@^7.0.0", "elasticsearch8": "npm:@elastic/elasticsearch@^8.0.0" diff --git a/packages/elasticsearch-store/package.json b/packages/elasticsearch-store/package.json index 40bf9cc9cb5..579ecdf8953 100644 --- a/packages/elasticsearch-store/package.json +++ b/packages/elasticsearch-store/package.json @@ -1,7 +1,7 @@ { "name": "elasticsearch-store", "displayName": "Elasticsearch Store", - "version": "0.83.1", + "version": "0.84.0", "description": "An API for managing an elasticsearch index, with versioning and migration support.", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/elasticsearch-store#readme", "bugs": { @@ -29,10 +29,10 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/data-mate": "^0.55.1", - "@terascope/data-types": "^0.49.1", - "@terascope/types": "^0.16.0", - "@terascope/utils": "^0.58.1", + "@terascope/data-mate": "^0.56.0", + "@terascope/data-types": "^0.50.0", + "@terascope/types": "^0.17.0", + "@terascope/utils": "^0.59.0", "ajv": "^6.12.6", "elasticsearch6": "npm:@elastic/elasticsearch@^6.7.0", "elasticsearch7": "npm:@elastic/elasticsearch@^7.0.0", @@ -41,7 +41,7 @@ "opensearch2": "npm:@opensearch-project/opensearch@^2.2.1", "setimmediate": "^1.0.5", "uuid": "^9.0.1", - "xlucene-translator": "^0.43.1" + "xlucene-translator": "^0.44.0" }, "devDependencies": { "@types/uuid": "^9.0.8" diff --git a/packages/generator-teraslice/package.json b/packages/generator-teraslice/package.json index 6d37985072c..b7ba454a22e 100644 --- a/packages/generator-teraslice/package.json +++ b/packages/generator-teraslice/package.json @@ -1,7 +1,7 @@ { "name": "generator-teraslice", "displayName": "Generator Teraslice", - "version": "0.37.1", + "version": "0.38.0", "description": "Generate teraslice related packages and code", "keywords": [ "teraslice", @@ -24,7 +24,7 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/utils": "^0.58.1", + "@terascope/utils": "^0.59.0", "chalk": "^4.1.2", "lodash": "^4.17.21", "yeoman-generator": "^5.8.0", diff --git a/packages/job-components/package.json b/packages/job-components/package.json index e7ec48dd583..bd949f24f4e 100644 --- a/packages/job-components/package.json +++ b/packages/job-components/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/job-components", "displayName": "Job Components", - "version": "0.73.2", + "version": "0.74.0", "description": "A teraslice library for validating jobs schemas, registering apis, and defining and running new Job APIs", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/job-components#readme", "bugs": { @@ -31,7 +31,7 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/utils": "^0.58.1", + "@terascope/utils": "^0.59.0", "convict": "^6.2.4", "convict-format-with-moment": "^6.2.0", "convict-format-with-validator": "^6.2.0", diff --git a/packages/job-components/src/execution-context/slicer.ts b/packages/job-components/src/execution-context/slicer.ts index c8643851a9d..a2735856a36 100644 --- a/packages/job-components/src/execution-context/slicer.ts +++ b/packages/job-components/src/execution-context/slicer.ts @@ -49,22 +49,6 @@ export class SlicerExecutionContext this.addOperation(op); this._resetMethodRegistry(); - - (async () => { - await config.context.apis.foundation.promMetrics.init({ - context: config.context, - logger: this.logger, - metrics_enabled_by_job: config.executionConfig.prom_metrics_enabled, - assignment: 'execution_controller', - port: config.executionConfig.prom_metrics_port, - default_metrics: config.executionConfig.prom_metrics_add_default, - labels: { - ex_id: this.exId, - job_id: this.jobId, - job_name: this.config.name, - } - }); - })(); } /** @@ -72,25 +56,6 @@ export class SlicerExecutionContext * @param recoveryData is the data to recover from */ async initialize(recoveryData?: SlicerRecoveryData[]): Promise { - await this.setupPromMetrics(); - await this.context.apis.foundation.promMetrics.addMetric( - 'info', - 'Information about Teraslice execution controller', - ['arch', 'clustering_type', 'name', 'node_version', 'platform', 'teraslice_version'], - 'gauge' - ); - this.context.apis.foundation.promMetrics.set( - 'info', - { - arch: this.context.arch, - clustering_type: this.context.sysconfig.teraslice.cluster_manager_type, - name: this.context.sysconfig.teraslice.name, - node_version: process.version, - platform: this.context.platform, - teraslice_version: this.config.teraslice_version - }, - 1 - ); return super.initialize(recoveryData); } @@ -118,26 +83,4 @@ export class SlicerExecutionContext onSliceComplete(result: SliceResult): void { this._runMethod('onSliceComplete', result); } - - /** - * Adds all prom metrics specific to the execution_controller. - * - * If trying to add a new metric for the execution_controller, it belongs here. - * @async - * @function setupPromMetrics - * @return {Promise} - * @link https://terascope.github.io/teraslice/docs/development/k8s#prometheus-metrics-api - */ - async setupPromMetrics() { - this.logger.info(`adding ${this.context.assignment} prom metrics...`); - await Promise.all([ - // All metrics go inside here - // this.context.apis.foundation.promMetrics.addMetric( - // 'example_metric', - // 'This is an example of adding a metric', - // ['example_label_1', 'Example_label_2'], - // 'gauge' - // ) - ]); - } } diff --git a/packages/job-components/src/execution-context/worker.ts b/packages/job-components/src/execution-context/worker.ts index ec1d86645db..c425c89b46a 100644 --- a/packages/job-components/src/execution-context/worker.ts +++ b/packages/job-components/src/execution-context/worker.ts @@ -113,44 +113,9 @@ export class WorkerExecutionContext return results; }); } - - (async () => { - await config.context.apis.foundation.promMetrics.init({ - context: config.context, - logger: this.logger, - metrics_enabled_by_job: config.executionConfig.prom_metrics_enabled, - assignment: 'worker', - port: config.executionConfig.prom_metrics_port, - default_metrics: config.executionConfig.prom_metrics_add_default, - labels: { - ex_id: this.exId, - job_id: this.jobId, - job_name: this.config.name, - } - }); - })(); } async initialize(): Promise { - await this.setupPromMetrics(); - await this.context.apis.foundation.promMetrics.addMetric( - 'worker_info', - 'Information about Teraslice worker', - ['arch', 'clustering_type', 'name', 'node_version', 'platform', 'teraslice_version'], - 'gauge' - ); - this.context.apis.foundation.promMetrics.set( - 'worker_info', - { - arch: this.context.arch, - clustering_type: this.context.sysconfig.teraslice.cluster_manager_type, - name: this.context.sysconfig.teraslice.name, - node_version: process.version, - platform: this.context.platform, - teraslice_version: this.config.teraslice_version - }, - 1 - ); await super.initialize(); this.status = 'idle'; } @@ -449,26 +414,4 @@ export class WorkerExecutionContext private get _slice() { return this.sliceState && this.sliceState.slice; } - - /** - * Adds all prom metrics specific to the worker. - * - * If trying to add a new metric for the worker, it belongs here. - * @async - * @function setupPromMetrics - * @return {Promise} - * @link https://terascope.github.io/teraslice/docs/development/k8s#prometheus-metrics-api - */ - async setupPromMetrics() { - this.logger.info(`adding ${this.context.assignment} prom metrics...`); - await Promise.all([ - // All metrics go inside here - // this.context.apis.foundation.promMetrics.addMetric( - // 'example_metric', - // 'This is an example of adding a metric', - // ['example_label_1', 'Example_label_2'], - // 'gauge' - // ) - ]); - } } diff --git a/packages/job-components/src/interfaces/context.ts b/packages/job-components/src/interfaces/context.ts index 567a193eccf..a4f38a8f1ce 100644 --- a/packages/job-components/src/interfaces/context.ts +++ b/packages/job-components/src/interfaces/context.ts @@ -1,5 +1,6 @@ import { EventEmitter } from 'events'; import { Logger } from '@terascope/utils'; +import { Terafoundation as tf } from '@terascope/types'; import { OpConfig } from './jobs'; import { ExecutionContextAPI } from '../execution-context'; @@ -105,7 +106,7 @@ export interface FoundationApis { getSystemEvents(): EventEmitter; getConnection(config: ConnectionConfig): { client: any }; createClient(config: ConnectionConfig): Promise<{ client: any }>; - promMetrics: PromMetrics + promMetrics: tf.PromMetrics } export interface LegacyFoundationApis { @@ -198,35 +199,3 @@ export interface ContextClusterConfig { } export type Assignment = 'assets_service'|'cluster_master'|'node_master'|'execution_controller'|'worker'; - -export interface PromMetricsInitConfig extends Omit { - context: Context, - logger: Logger, - metrics_enabled_by_job?: boolean, - port?: number - default_metrics?: boolean -} -export interface PromMetricsAPIConfig { - assignment: string - port: number - default_metrics: boolean, - labels?: Record, - prefix?: string -} - -export interface PromMetrics { - init: (config: PromMetricsInitConfig) => Promise; - set: (name: string, labels: Record, value: number) => void; - inc: (name: string, labelValues: Record, value: number) => void; - dec: (name: string, labelValues: Record, value: number) => void; - observe: (name: string, labelValues: Record, value: number) => void; - addMetric: (name: string, help: string, labelNames: Array, type: 'gauge' | 'counter' | 'histogram', - buckets?: Array) => Promise; - addSummary: (name: string, help: string, labelNames: Array, - maxAgeSeconds?: number, ageBuckets?: number, - percentiles?: Array) => Promise; - hasMetric: (name: string) => boolean; - deleteMetric: (name: string) => Promise; - verifyAPI: () => boolean; - shutdown: () => Promise; -} diff --git a/packages/job-components/src/test-helpers.ts b/packages/job-components/src/test-helpers.ts index 0174289a51a..91270f73901 100644 --- a/packages/job-components/src/test-helpers.ts +++ b/packages/job-components/src/test-helpers.ts @@ -5,6 +5,7 @@ import { isFunction, debugLogger, Logger, makeISODate } from '@terascope/utils'; +import { Terafoundation as tf } from '@terascope/types'; import * as i from './interfaces'; function newId(prefix: string): string { @@ -119,10 +120,11 @@ export type MockPromMetrics = Record, readonly percentiles?: Array, readonly ageBuckets?: number, - readonly maxAgeSeconds?: number - readonly metric: 'Gauge' | 'Counter' | 'Histogram' | 'Summary', + readonly maxAgeSeconds?: number, readonly functions: Set, labels: Record + collect?: tf.CollectFunction | tf.CollectFunction + | tf.CollectFunction | tf.CollectFunction, }>; type GetKeyOpts = { @@ -328,21 +330,15 @@ export class TestContext implements i.Context { return events; }, promMetrics: { - async init(config: i.PromMetricsInitConfig) { - const { terafoundation, teraslice } = config.context.sysconfig; - const metricsEnabledInTF = terafoundation.prom_metrics_enabled; + async init(config: tf.PromMetricsInitConfig) { + const { job_prom_metrics_enabled, tf_prom_metrics_enabled } = config; if (ctx.mockPromMetrics) { throw new Error('Prom metrics API cannot be initialized more than once.'); } - if (teraslice.cluster_manager_type === 'native') { - logger.warn('Skipping PromMetricsAPI initialization: incompatible with native clustering.'); - return false; - } - - if (config.metrics_enabled_by_job === true - || (config.metrics_enabled_by_job === undefined && metricsEnabledInTF)) { + if (job_prom_metrics_enabled === true + || (job_prom_metrics_enabled === undefined && tf_prom_metrics_enabled)) { ctx.mockPromMetrics = {}; return true; } @@ -352,7 +348,7 @@ export class TestContext implements i.Context { set(name: string, labelValues: Record, value: number): void { if (ctx.mockPromMetrics) { const metric = ctx.mockPromMetrics[name]; - if (!metric || !metric.functions || !metric.metric) { + if (!metric || !metric.functions) { throw new Error(`Metric ${name} is not setup`); } if (metric.functions.has('inc')) { @@ -364,7 +360,7 @@ export class TestContext implements i.Context { inc(name: string, labelValues: Record, value: number): void { if (ctx.mockPromMetrics) { const metric = ctx.mockPromMetrics[name]; - if (!metric || !metric.functions || !metric.metric) { + if (!metric || !metric.functions) { throw new Error(`Metric ${name} is not setup`); } if (metric.functions.has('inc')) { @@ -380,7 +376,7 @@ export class TestContext implements i.Context { dec(name: string, labelValues: Record, value: number): void { if (ctx.mockPromMetrics) { const metric = ctx.mockPromMetrics[name]; - if (!metric || !metric.functions || !metric.metric) { + if (!metric || !metric.functions) { throw new Error(`Metric ${name} is not setup`); } if (metric.functions.has('dec')) { @@ -400,7 +396,7 @@ export class TestContext implements i.Context { ): void { if (ctx.mockPromMetrics) { const metric = ctx.mockPromMetrics[name]; - if (!metric || !metric.functions || !metric.metric) { + if (!metric || !metric.functions) { throw new Error(`Metric ${name} is not setup`); } @@ -424,46 +420,64 @@ export class TestContext implements i.Context { } } }, - async addMetric( + async addGauge( name: string, help: string, labelNames: Array, - type: 'gauge' | 'counter' | 'histogram', - buckets?: Array + collect?: tf.CollectFunction ): Promise { if (ctx.mockPromMetrics) { if (!this.hasMetric(name)) { - if (type === 'gauge') { - ctx.mockPromMetrics[name] = { - name, - help, - labelNames, - metric: 'Gauge', - functions: new Set(['inc', 'dec', 'set']), - labels: {} - }; - } - if (type === 'counter') { - ctx.mockPromMetrics[name] = { - name, - help, - labelNames, - metric: 'Counter', - functions: new Set(['inc', 'dec']), - labels: {} - }; - } - if (type === 'histogram') { - ctx.mockPromMetrics[name] = { - name, - help, - labelNames, - buckets, - metric: 'Histogram', - functions: new Set(['observe']), - labels: {} - }; - } + ctx.mockPromMetrics[name] = { + name, + help, + labelNames, + collect, + functions: new Set(['inc', 'dec', 'set']), + labels: {} + }; + } else { + logger.info(`metric ${name} already defined in metric list`); + } + } + }, + async addCounter( + name: string, + help: string, + labelNames: Array, + collect?: tf.CollectFunction + ): Promise { + if (ctx.mockPromMetrics) { + if (!this.hasMetric(name)) { + ctx.mockPromMetrics[name] = { + name, + help, + labelNames, + collect, + functions: new Set(['inc']), + labels: {} + }; + } else { + logger.info(`metric ${name} already defined in metric list`); + } + } + }, + async addHistogram( + name: string, + help: string, + labelNames: Array, + collect?: tf.CollectFunction + ): Promise { + if (ctx.mockPromMetrics) { + if (!this.hasMetric(name)) { + ctx.mockPromMetrics[name] = { + name, + help, + labelNames, + collect, + functions: new Set(['observe']), + labels: {} + }; } else { logger.info(`metric ${name} already defined in metric list`); } @@ -473,22 +487,27 @@ export class TestContext implements i.Context { name: string, help: string, labelNames: Array, + collect?: tf.CollectFunction, maxAgeSeconds = 600, ageBuckets = 5, percentiles: Array = [0.01, 0.1, 0.9, 0.99] ): Promise { if (ctx.mockPromMetrics) { - ctx.mockPromMetrics[name] = { - name, - help, - labelNames, - maxAgeSeconds, - ageBuckets, - percentiles, - metric: 'Summary', - functions: new Set(['observe']), - labels: {} - }; + if (!this.hasMetric(name)) { + ctx.mockPromMetrics[name] = { + name, + help, + labelNames, + collect, + maxAgeSeconds, + ageBuckets, + percentiles, + functions: new Set(['observe']), + labels: {} + }; + } else { + logger.info(`metric ${name} already defined in metric list`); + } } }, hasMetric(name: string): boolean { diff --git a/packages/job-components/test/test-helpers-spec.ts b/packages/job-components/test/test-helpers-spec.ts index ff9d4425577..45ea73add4f 100644 --- a/packages/job-components/test/test-helpers-spec.ts +++ b/packages/job-components/test/test-helpers-spec.ts @@ -185,11 +185,12 @@ describe('Test Helpers', () => { describe('MockPromMetrics', () => { const context = new TestContext('test-prom-metrics'); context.sysconfig.teraslice.cluster_manager_type = 'kubernetes'; - context.sysconfig.terafoundation.prom_metrics_enabled = true; const config = { - context, + assignment: 'cluster-master', logger: debugLogger('test-helpers-spec-logger'), - assignment: 'cluster-master' + tf_prom_metrics_enabled: true, + tf_prom_metrics_port: 3333, + tf_prom_metrics_add_default: false, }; it('should be able to init a mock prom_metrics_api', async () => { @@ -201,14 +202,17 @@ describe('Test Helpers', () => { await expect(context.apis.foundation.promMetrics.init(config)).rejects.toThrow('Prom metrics API cannot be initialized more than once.'); }); - it('should add and delete metric', async () => { - await context.apis.foundation.promMetrics.addMetric('test_counter', 'test_counter help string', ['uuid'], 'counter'); + it('should add, inc and delete counter', async () => { + await context.apis.foundation.promMetrics.addCounter('test_counter', 'test_counter help string', ['uuid'], function collect() { + this.inc(); + }); + context.apis.foundation.promMetrics.inc('test_counter', {}, 1); expect(context.apis.foundation.promMetrics.hasMetric('test_counter')).toBe(true); expect(await context.apis.foundation.promMetrics.deleteMetric('test_counter')).toBe(true); }); - it('should inc, dec, and set metric', async () => { - await context.apis.foundation.promMetrics.addMetric('test_gauge', 'test_gauge help string', ['uuid'], 'gauge'); + it('should inc, dec, and set gauge', async () => { + await context.apis.foundation.promMetrics.addGauge('test_gauge', 'test_gauge help string', ['uuid']); context.apis.foundation.promMetrics.set('test_gauge', { uuid: '437Ev89h' }, 10); context.apis.foundation.promMetrics.inc('test_gauge', { uuid: '437Ev89h' }, 1); context.apis.foundation.promMetrics.dec('test_gauge', { uuid: '437Ev89h' }, 2); @@ -238,7 +242,7 @@ describe('Test Helpers', () => { }); it('should add and observe histogram', async () => { - await context.apis.foundation.promMetrics.addMetric('test_histogram', 'test_histogram help string', ['uuid'], 'histogram'); + await context.apis.foundation.promMetrics.addHistogram('test_histogram', 'test_histogram help string', ['uuid']); context.apis.foundation.promMetrics.observe('test_histogram', { uuid: 'dEF4Kby6' }, 10); context.apis.foundation.promMetrics.observe('test_histogram', { uuid: 'dEF4Kby6' }, 30); context.apis.foundation.promMetrics.observe('test_histogram', { uuid: 'dEF4Kby6' }, 2); diff --git a/packages/scripts/package.json b/packages/scripts/package.json index c08897f5810..bf0be4bae96 100644 --- a/packages/scripts/package.json +++ b/packages/scripts/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/scripts", "displayName": "Scripts", - "version": "0.76.1", + "version": "0.77.0", "description": "A collection of terascope monorepo scripts", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/scripts#readme", "bugs": { @@ -32,7 +32,7 @@ }, "dependencies": { "@kubernetes/client-node": "^0.20.0", - "@terascope/utils": "^0.58.1", + "@terascope/utils": "^0.59.0", "codecov": "^3.8.3", "execa": "^5.1.0", "fs-extra": "^11.2.0", diff --git a/packages/terafoundation/package.json b/packages/terafoundation/package.json index 6eb38abc4d7..ffe6e391cff 100644 --- a/packages/terafoundation/package.json +++ b/packages/terafoundation/package.json @@ -1,7 +1,7 @@ { "name": "terafoundation", "displayName": "Terafoundation", - "version": "0.62.1", + "version": "0.63.0", "description": "A Clustering and Foundation tool for Terascope Tools", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/terafoundation#readme", "bugs": { @@ -28,15 +28,15 @@ }, "dependencies": { "@terascope/file-asset-apis": "^0.13.0", - "@terascope/types": "^0.16.0", - "@terascope/utils": "^0.58.1", + "@terascope/types": "^0.17.0", + "@terascope/utils": "^0.59.0", "bluebird": "^3.7.2", "bunyan": "^1.8.15", "convict": "^6.2.4", "convict-format-with-moment": "^6.2.0", "convict-format-with-validator": "^6.2.0", "elasticsearch": "^15.4.1", - "elasticsearch-store": "^0.83.1", + "elasticsearch-store": "^0.84.0", "express": "^4.19.2", "js-yaml": "^4.1.0", "nanoid": "^3.3.4", diff --git a/packages/terafoundation/src/api/index.ts b/packages/terafoundation/src/api/index.ts index 08d90dfe6f7..7cffc4a59f1 100644 --- a/packages/terafoundation/src/api/index.ts +++ b/packages/terafoundation/src/api/index.ts @@ -153,7 +153,7 @@ export default function registerApis(context: i.FoundationContext): void { return workers; }, - promMetrics: new PromMetrics(context, context.logger) + promMetrics: new PromMetrics(context.name, context.logger) }; function _registerFoundationAPIs() { registerAPI('foundation', foundationApis); @@ -208,10 +208,11 @@ export default function registerApis(context: i.FoundationContext): void { } if (funcName === 'hasMetric' || funcName === 'deleteMetric') { return () => false; } if ( - funcName === 'set' || funcName === 'addMetric' + funcName === 'set' || funcName === 'addGauge' + || funcName === 'addCounter' || funcName === 'addHistogram' || funcName === 'addSummary' || funcName === 'inc' || funcName === 'dec' || funcName === 'observe' - || funcName === 'shutdown' + || funcName === 'getDefaultLabels' || funcName === 'shutdown' ) { return () => { /// return empty function diff --git a/packages/terafoundation/src/api/prom-metrics/exporter.ts b/packages/terafoundation/src/api/prom-metrics/exporter.ts index 247fbb7cc2d..c4ac2f290e1 100644 --- a/packages/terafoundation/src/api/prom-metrics/exporter.ts +++ b/packages/terafoundation/src/api/prom-metrics/exporter.ts @@ -2,7 +2,7 @@ import promClient from 'prom-client'; import express, { Request, Response } from 'express'; import { Server } from 'http'; import { Logger } from '@terascope/utils'; -import { PromMetricsAPIConfig } from '../../interfaces'; +import { Terafoundation as tf } from '@terascope/types'; export type CloseExporter = () => void; @@ -15,7 +15,7 @@ export default class Exporter { } async create( - promMetricsAPIConfig: PromMetricsAPIConfig + promMetricsAPIConfig: tf.PromMetricsAPIConfig ) { try { if (promMetricsAPIConfig.default_metrics) { diff --git a/packages/terafoundation/src/api/prom-metrics/prom-metrics-api.ts b/packages/terafoundation/src/api/prom-metrics/prom-metrics-api.ts index f298e614b2d..97a6bb8a890 100644 --- a/packages/terafoundation/src/api/prom-metrics/prom-metrics-api.ts +++ b/packages/terafoundation/src/api/prom-metrics/prom-metrics-api.ts @@ -2,30 +2,31 @@ import { Logger } from '@terascope/utils'; import os from 'os'; import { - Gauge, Counter, Histogram, Summary + Gauge, Counter, Histogram, Summary, + CollectFunction } from 'prom-client'; -import * as i from '../../interfaces'; +import { Terafoundation as tf } from '@terascope/types'; import Exporter from './exporter'; export class PromMetrics { - readonly metricList!: i.MetricList; + readonly metricList!: tf.MetricList; default_labels!: Record; prefix: string; private metricExporter!: Exporter; - context: i.FoundationContext; - apiConfig: i.PromMetricsAPIConfig; + name: string; + apiConfig: tf.PromMetricsAPIConfig; apiRunning: boolean; logger: Logger; constructor( - context: i.FoundationContext, + name: string, logger: Logger, ) { - this.context = context; + this.name = name; this.apiRunning = false; - this.apiConfig = {} as i.PromMetricsAPIConfig; + this.apiConfig = {} as tf.PromMetricsAPIConfig; this.prefix = ''; this.logger = logger.child({ module: 'prom_metrics' }); this.metricList = {}; @@ -38,38 +39,35 @@ export class PromMetrics { * of terafoundation. This allows jobs to set different metrics configurations than the master. * @returns {Promise} Was the API initialized */ - async init(config: i.PromMetricsInitConfig) { - const { terafoundation, teraslice } = config.context.sysconfig; - const metricsEnabledInTF = terafoundation.prom_metrics_enabled; - const portToUse = config.port || terafoundation.prom_metrics_port; - - if (teraslice.cluster_manager_type === 'native') { - this.logger.warn('Skipping PromMetricsAPI initialization: incompatible with native clustering.'); - return false; - } + async init(config: tf.PromMetricsInitConfig) { + const { + assignment, job_prom_metrics_add_default, job_prom_metrics_enabled, + job_prom_metrics_port, tf_prom_metrics_add_default, tf_prom_metrics_enabled, + tf_prom_metrics_port, labels, prefix + } = config; + + const portToUse = job_prom_metrics_port || tf_prom_metrics_port; + + const useDefaultMetrics = job_prom_metrics_add_default !== undefined + ? job_prom_metrics_add_default + : tf_prom_metrics_add_default; - // If prom_metrics_add_default is defined in jobSpec use that value. - // If not use the terafoundation value. - const useDefaultMetrics = config.default_metrics !== undefined - ? config.default_metrics - : terafoundation.prom_metrics_add_default; - - // If prom_metrics_enabled is true in jobConfig, or if not specified in - // jobSpec and true in terafoundation, then we enable metrics. - if (config.metrics_enabled_by_job === true - || (config.metrics_enabled_by_job === undefined && metricsEnabledInTF)) { - const apiConfig: i.PromMetricsAPIConfig = { - assignment: config.assignment, + // If job_prom_metrics_enabled is true, or if not specified and + // tf_prom_metrics_enabled is true, then we enable metrics. + if (job_prom_metrics_enabled === true + || (job_prom_metrics_enabled === undefined && tf_prom_metrics_enabled)) { + const apiConfig: tf.PromMetricsAPIConfig = { + assignment, port: portToUse, default_metrics: useDefaultMetrics, - labels: config.labels, - prefix: config.prefix + labels, + prefix }; this.prefix = apiConfig.prefix || `teraslice_${apiConfig.assignment}_`; this.default_labels = { - name: this.context.sysconfig.teraslice.name, + name: this.name, assignment: apiConfig.assignment, ...apiConfig.labels }; @@ -175,74 +173,80 @@ export class PromMetrics { } /** - * [addMetric (define) new metric] + * [addGauge (define) new gauge] * @param {string} name [metric name] * @param {string} help [metric help] * @param {Array} labelsNames [list of label names] - * @param {'gauge' | 'counter' | 'histogram'} type [gauge,counter,histogram,or summary] - * @param {Array} buckets [default buckets] + * @param {CollectFunction} collectFn [fn invoked when metrics endpoint scraped] * @return {Promise} */ - async addMetric( + async addGauge( name: string, help: string, labelsNames: Array, - type: 'gauge' | 'counter' | 'histogram', - buckets: Array = [0.1, 5, 15, 50, 100, 500] + collectFn?: CollectFunction, ): Promise { if (!this.hasMetric(name)) { const fullname = this.prefix + name; - if (type === 'gauge') { - this.metricList[name] = this._createGaugeMetric( - fullname, help, labelsNames.concat(Object.keys(this.default_labels)) - ); - } - if (type === 'counter') { - this.metricList[name] = this._createCounterMetric( - fullname, help, labelsNames.concat(Object.keys(this.default_labels)) - ); - } - if (type === 'histogram') { - this.metricList[name] = this._createHistogramMetric( - fullname, help, labelsNames.concat(Object.keys(this.default_labels)), buckets - ); - } + this.metricList[name] = this._createGaugeMetric( + fullname, help, labelsNames.concat(Object.keys(this.default_labels)), collectFn + ); } else { this.logger.info(`metric ${name} already defined in metric list`); } } /** - * [hasMetric check if metricList contains metric] + * [addCounter (define) new counter metric] * @param {string} name [metric name] - * @return {boolean} + * @param {string} help [metric help] + * @param {Array} labelsNames [list of label names] + * @param {CollectFunction} collectFn [fn invoked when metrics endpoint scraped] + * @return {Promise} */ - - hasMetric(name: string): boolean { - return (name in this.metricList); + async addCounter( + name: string, + help: string, + labelsNames: Array, + collectFn?: CollectFunction, + ): Promise { + if (!this.hasMetric(name)) { + const fullname = this.prefix + name; + this.metricList[name] = this._createCounterMetric( + fullname, help, labelsNames.concat(Object.keys(this.default_labels)), collectFn + ); + } else { + this.logger.info(`metric ${name} already defined in metric list`); + } } /** - * [deleteMetric delete metric from metricList] + * [addHistogram (define) new histogram metric] * @param {string} name [metric name] - * @return {Promise} [was the metric successfully deleted] + * @param {string} help [metric help] + * @param {Array} labelsNames [list of label names] + * @param {CollectFunction} collectFn [fn invoked when metrics endpoint scraped] + * @return {Promise} */ - async deleteMetric(name: string): Promise { - let deleted = false; - const fullname = this.prefix + name; - this.logger.info(`delete metric ${fullname}`); - if (this.hasMetric(name)) { - deleted = delete this.metricList[name]; - try { - await this.metricExporter.deleteMetric(fullname); - } catch (err) { - deleted = false; - throw new Error(`unable to delete metric ${fullname} from exporter`); - } + async addHistogram( + name: string, + help: string, + labelsNames: Array, + collectFn?: CollectFunction, + buckets: Array = [0.1, 5, 15, 50, 100, 500], + ): Promise { + if (!this.hasMetric(name)) { + const fullname = this.prefix + name; + this.metricList[name] = this._createHistogramMetric( + fullname, + help, + labelsNames.concat(Object.keys(this.default_labels)), + buckets, + collectFn + ); } else { - throw new Error(`metric ${name} not defined in metric list`); + this.logger.info(`metric ${name} already defined in metric list`); } - return deleted; } /** @@ -250,14 +254,17 @@ export class PromMetrics { * @param {string} name [metric name] * @param {string} help [metric help] * @param {Array} labelsNames [list of label names] + * @param {CollectFunction} collectFn [fn invoked when metrics endpoint scraped] * @param {Array} percentiles [metric percentiles, default[0.01, 0.1, 0.9, 0.99] ] * @param {number} maxAgeSeconds [how old a bucket can be before it is reset ] * @param {number} ageBuckets [how many buckets for sliding window ] * @return {Promise} */ - async addSummary(name: string, + async addSummary( + name: string, help: string, labelsNames: Array, + collectFn?: CollectFunction, maxAgeSeconds = 600, ageBuckets = 5, percentiles: Array = [0.01, 0.1, 0.9, 0.99] @@ -270,53 +277,98 @@ export class PromMetrics { labelsNames.concat(Object.keys(this.default_labels)), percentiles, maxAgeSeconds, - ageBuckets + ageBuckets, + collectFn, ); } } + /** + * [hasMetric check if metricList contains metric] + * @param {string} name [metric name] + * @return {boolean} + */ + + hasMetric(name: string): boolean { + return (name in this.metricList); + } + + /** + * [deleteMetric delete metric from metricList] + * @param {string} name [metric name] + * @return {Promise} [was the metric successfully deleted] + */ + async deleteMetric(name: string): Promise { + let deleted = false; + const fullname = this.prefix + name; + this.logger.info(`delete metric ${fullname}`); + if (this.hasMetric(name)) { + deleted = delete this.metricList[name]; + try { + await this.metricExporter.deleteMetric(fullname); + } catch (err) { + deleted = false; + throw new Error(`unable to delete metric ${fullname} from exporter`); + } + } else { + throw new Error(`metric ${name} not defined in metric list`); + } + return deleted; + } + private _createGaugeMetric(name: string, help: string, - labelsNames: Array): any { + labelsNames: Array, collect?: CollectFunction): any { const gauge = new Gauge({ name, help, labelNames: labelsNames, + collect }); return { name, metric: gauge, functions: new Set(['inc', 'dec', 'set']) }; } private _createCounterMetric(name: string, help: string, - labelsNames: Array): any { + labelsNames: Array, collect?: CollectFunction): any { const counter = new Counter({ name, help, labelNames: labelsNames, + collect }); return { name, metric: counter, functions: new Set(['inc']) }; } private _createHistogramMetric(name: string, help: string, labelsNames: Array, - buckets: Array): any { + buckets: Array, collect?: CollectFunction): any { const histogram = new Histogram({ name, help, labelNames: labelsNames, - buckets + buckets, + collect }); return { name, metric: histogram, functions: new Set(['observe']) }; } - private _createSummaryMetric(name: string, help: string, labelsNames: Array, - percentiles: Array, ageBuckets: number, maxAgeSeconds: number): any { - const histogram = new Summary({ + private _createSummaryMetric( + name: string, + help: string, + labelsNames: Array, + percentiles: Array, + ageBuckets: number, + maxAgeSeconds: number, + collect?: CollectFunction, + ): any { + const summary = new Summary({ name, help, labelNames: labelsNames, percentiles, maxAgeSeconds, - ageBuckets + ageBuckets, + collect }); - return { name, metric: histogram, functions: new Set(['observe']) }; + return { name, metric: summary, functions: new Set(['observe']) }; } private _setPodName(): void { @@ -327,7 +379,7 @@ export class PromMetrics { } } - private async createAPI(apiConfig: i.PromMetricsAPIConfig) { + private async createAPI(apiConfig: tf.PromMetricsAPIConfig) { this._setPodName(); try { if (!this.metricExporter) { @@ -347,6 +399,10 @@ export class PromMetrics { this.apiRunning = true; } + getDefaultLabels() { + return this.default_labels; + } + verifyAPI(): boolean { return this.apiRunning; } diff --git a/packages/terafoundation/src/interfaces.ts b/packages/terafoundation/src/interfaces.ts index 0231e7490dc..82e195231c8 100644 --- a/packages/terafoundation/src/interfaces.ts +++ b/packages/terafoundation/src/interfaces.ts @@ -1,13 +1,11 @@ import { Format } from 'convict'; -import type { - Gauge, Counter, Histogram, Summary -} from 'prom-client'; import { EventEmitter } from 'events'; import { Logger, Overwrite } from '@terascope/utils'; import { Cluster as NodeJSCluster, Worker as NodeJSWorker } from 'cluster'; +import { Terafoundation as tf } from '@terascope/types'; export type FoundationConfig< S = Record, @@ -59,7 +57,7 @@ export interface FoundationAPIs { getConnection(config: ConnectionConfig): { client: any }; createClient(config: ConnectionConfig): Promise<{ client: any }>; startWorkers(num: number, envOptions: Record): void; - promMetrics: PromMetrics + promMetrics: tf.PromMetrics } export interface LegacyFoundationApis { @@ -134,41 +132,3 @@ export type FoundationContext< export type ParsedArgs = { configfile: FoundationSysConfig; }; - -export interface PromMetricsInitConfig extends Omit { - context: FoundationContext, - logger: Logger, - metrics_enabled_by_job?: boolean, - port?: number - default_metrics?: boolean -} -export interface PromMetricsAPIConfig { - assignment: string - port: number - default_metrics: boolean, - labels?: Record, - prefix?: string -} - -export type MetricList = Record | Counter | Histogram | Summary, - readonly functions?: Set -}>; - -export interface PromMetrics { - init: (config: PromMetricsInitConfig) => Promise; - set: (name: string, labels: Record, value: number) => void; - inc: (name: string, labelValues: Record, value: number) => void; - dec: (name: string, labelValues: Record, value: number) => void; - observe: (name: string, labelValues: Record, value: number) => void; - addMetric: (name: string, help: string, labelNames: Array, type: 'gauge' | 'counter' | 'histogram', - buckets?: Array) => Promise; - addSummary: (name: string, help: string, labelNames: Array, - ageBuckets: number, maxAgeSeconds: number, - percentiles: Array) => Promise; - hasMetric: (name: string) => boolean; - deleteMetric: (name: string) => Promise; - verifyAPI: () => boolean; - shutdown: () => Promise; -} diff --git a/packages/terafoundation/src/test-context.ts b/packages/terafoundation/src/test-context.ts index 71948d53ec6..95ed494caa8 100644 --- a/packages/terafoundation/src/test-context.ts +++ b/packages/terafoundation/src/test-context.ts @@ -171,7 +171,7 @@ export class TestContext< return client; }; - this.apis.foundation.promMetrics = new PromMetrics(ctx, ctx.logger); + this.apis.foundation.promMetrics = new PromMetrics(ctx.name, ctx.logger); this.apis.setTestClients = (clients: TestClientConfig[] = []) => { clients.forEach((clientConfig) => { diff --git a/packages/terafoundation/test/apis/api-spec.ts b/packages/terafoundation/test/apis/api-spec.ts index 47e6eb1f7fe..aecd68a3caa 100644 --- a/packages/terafoundation/test/apis/api-spec.ts +++ b/packages/terafoundation/test/apis/api-spec.ts @@ -60,7 +60,21 @@ describe('apis module', () => { expect(context.apis.foundation.getConnection).toBeDefined(); expect(context.apis.foundation.startWorkers).toBeDefined(); expect(context.apis.foundation.createClient).toBeDefined(); + expect(context.apis.foundation.promMetrics).toBeDefined(); expect(context.apis.foundation.promMetrics.init).toBeDefined(); + expect(context.apis.foundation.promMetrics.set).toBeDefined(); + expect(context.apis.foundation.promMetrics.inc).toBeDefined(); + expect(context.apis.foundation.promMetrics.dec).toBeDefined(); + expect(context.apis.foundation.promMetrics.observe).toBeDefined(); + expect(context.apis.foundation.promMetrics.addGauge).toBeDefined(); + expect(context.apis.foundation.promMetrics.addCounter).toBeDefined(); + expect(context.apis.foundation.promMetrics.addHistogram).toBeDefined(); + expect(context.apis.foundation.promMetrics.addSummary).toBeDefined(); + expect(context.apis.foundation.promMetrics.hasMetric).toBeDefined(); + expect(context.apis.foundation.promMetrics.deleteMetric).toBeDefined(); + expect(context.apis.foundation.promMetrics.verifyAPI).toBeDefined(); + expect(context.apis.foundation.promMetrics.shutdown).toBeDefined(); + expect(context.apis.foundation.promMetrics.getDefaultLabels).toBeDefined(); expect(typeof context.apis.foundation.makeLogger).toBe('function'); expect(typeof context.apis.foundation.getSystemEvents).toBe('function'); @@ -73,11 +87,14 @@ describe('apis module', () => { expect(typeof context.apis.foundation.promMetrics.inc).toBe('function'); expect(typeof context.apis.foundation.promMetrics.dec).toBe('function'); expect(typeof context.apis.foundation.promMetrics.observe).toBe('function'); - expect(typeof context.apis.foundation.promMetrics.addMetric).toBe('function'); + expect(typeof context.apis.foundation.promMetrics.addGauge).toBe('function'); + expect(typeof context.apis.foundation.promMetrics.addCounter).toBe('function'); + expect(typeof context.apis.foundation.promMetrics.addHistogram).toBe('function'); expect(typeof context.apis.foundation.promMetrics.addSummary).toBe('function'); expect(typeof context.apis.foundation.promMetrics.hasMetric).toBe('function'); expect(typeof context.apis.foundation.promMetrics.deleteMetric).toBe('function'); expect(typeof context.apis.foundation.promMetrics.verifyAPI).toBe('function'); expect(typeof context.apis.foundation.promMetrics.shutdown).toBe('function'); + expect(typeof context.apis.foundation.promMetrics.getDefaultLabels).toBe('function'); }); }); diff --git a/packages/terafoundation/test/apis/exporter-spec.ts b/packages/terafoundation/test/apis/exporter-spec.ts index 405a181c4a1..10e76658aeb 100644 --- a/packages/terafoundation/test/apis/exporter-spec.ts +++ b/packages/terafoundation/test/apis/exporter-spec.ts @@ -3,8 +3,8 @@ import 'jest-extended'; import got from 'got'; import { Counter } from 'prom-client'; import { debugLogger } from '@terascope/utils'; +import { Terafoundation as tf } from '@terascope/types'; import Exporter from '../../src/api/prom-metrics/exporter'; -import { PromMetricsAPIConfig } from '../../src'; describe('prometheus exporter', () => { let exporter: Exporter; @@ -13,7 +13,7 @@ describe('prometheus exporter', () => { exporter = new Exporter(logger); }); describe('create', () => { - const config: PromMetricsAPIConfig = { + const config: tf.PromMetricsAPIConfig = { assignment: 'worker', port: 3344, default_metrics: false, diff --git a/packages/terafoundation/test/apis/prom-metrics-spec.ts b/packages/terafoundation/test/apis/prom-metrics-spec.ts index 53e184f9786..35f45041b17 100644 --- a/packages/terafoundation/test/apis/prom-metrics-spec.ts +++ b/packages/terafoundation/test/apis/prom-metrics-spec.ts @@ -2,300 +2,283 @@ import 'jest-extended'; import { debugLogger } from '@terascope/utils'; import got from 'got'; - +import { + Counter, Gauge, Histogram, Summary +} from 'prom-client'; import api from '../../src/api'; describe('promMetrics foundation API', () => { describe('init', () => { - describe('in native clustering', () => { - const context = { - sysconfig: { - terafoundation: { - log_level: 'debug', - prom_metrics_enabled: true, - prom_metrics_port: 3333, - prom_metrics_add_default: true + describe('with prom metrics enabled in terafoundation', () => { + describe('with enable_prom_metrics undefined in jobConfig', () => { + const context = { + sysconfig: { + terafoundation: { + log_level: 'debug', + prom_metrics_enabled: true, + prom_metrics_port: 3333, + prom_metrics_add_default: true + }, + teraslice: { + cluster_manager_type: 'kubernetes', + } }, - teraslice: { - cluster_manager_type: 'native' - } - } - } as any; - - const config = { - context, - logger: debugLogger('prom-metrics-spec-logger'), - assignment: 'cluster-master' - }; - beforeAll(() => { - // This sets up the API endpoints in the context. - api(context); - context.logger = debugLogger('terafoundation-tests'); - }); - afterAll(async () => { - await context.apis.foundation.promMetrics.shutdown(); + name: 'tera-test' + } as any; + + const { terafoundation } = context.sysconfig; + const config = { + tf_prom_metrics_enabled: terafoundation.prom_metrics_enabled, + tf_prom_metrics_port: terafoundation.prom_metrics_port, + tf_prom_metrics_add_default: terafoundation.prom_metrics_add_default, + logger: debugLogger('prom-metrics-spec-logger'), + assignment: 'worker' + }; + + beforeAll(() => { + // This sets up the API endpoints in the context. + api(context); + context.logger = debugLogger('terafoundation-tests'); + }); + + afterAll(async () => { + await context.apis.foundation.promMetrics.shutdown(); + }); + + it('should initialize a promMetricsAPI', async () => { + const result = await context.apis.foundation.promMetrics.init(config); + expect(result).toBe(true); + }); + + it('should be able to verifyApi', async () => { + const apiExists = await context.apis.foundation.promMetrics.verifyAPI(); + expect(apiExists).toBe(true); + }); + + it('should throw an error if promMetricsAPI is already initialized', async () => { + expect(() => context.apis.foundation.promMetrics.init(config)) + .toThrow('Prom metrics API cannot be initialized more than once.'); + }); }); - it('should not initialize a promMetricsAPI', async () => { - const result = await context.apis.foundation.promMetrics.init(config); - expect(result).toBe(false); + describe('with prom metrics disabled in jobConfig', () => { + const context = { + sysconfig: { + terafoundation: { + log_level: 'debug', + prom_metrics_enabled: true, + prom_metrics_port: 3333, + prom_metrics_add_default: true + }, + teraslice: { + cluster_manager_type: 'kubernetes', + } + }, + name: 'tera-test' + } as any; + + const { terafoundation } = context.sysconfig; + const config = { + tf_prom_metrics_enabled: terafoundation.prom_metrics_enabled, + tf_prom_metrics_port: terafoundation.prom_metrics_port, + tf_prom_metrics_add_default: terafoundation.prom_metrics_add_default, + logger: debugLogger('prom-metrics-spec-logger'), + assignment: 'worker', + job_prom_metrics_enabled: false + }; + + beforeAll(() => { + // This sets up the API endpoints in the context. + api(context); + context.logger = debugLogger('terafoundation-tests'); + }); + + afterAll(async () => { + await context.apis.foundation.promMetrics.shutdown(); + }); + + it('should not initialize a promMetricsAPI', async () => { + const result = await context.apis.foundation.promMetrics.init(config); + expect(result).toBe(false); + }); }); - it('should not throw an error when making a call to uninitialized promMetricsAPI', async () => { - expect(async () => context.apis.foundation.promMetrics.addMetric('native_counter', 'help message', ['uuid'], 'counter')).not.toThrow(); - expect(async () => context.apis.foundation.promMetrics.inc('native_counter', { uuid: 'fsd784bf' }, 1)).not.toThrow(); - expect(context.apis.foundation.promMetrics.hasMetric('native_counter')).toBe(false); + describe('with prom metrics enabled in jobConfig', () => { + const context = { + sysconfig: { + terafoundation: { + log_level: 'debug', + prom_metrics_enabled: true, + prom_metrics_port: 3333, + prom_metrics_add_default: true + }, + teraslice: { + cluster_manager_type: 'kubernetes', + } + }, + name: 'tera-test' + } as any; + + const { terafoundation } = context.sysconfig; + const config = { + tf_prom_metrics_enabled: terafoundation.prom_metrics_enabled, + tf_prom_metrics_port: terafoundation.prom_metrics_port, + tf_prom_metrics_add_default: terafoundation.prom_metrics_add_default, + logger: debugLogger('prom-metrics-spec-logger'), + assignment: 'worker', + job_prom_metrics_enabled: true + }; + + beforeAll(() => { + // This sets up the API endpoints in the context. + api(context); + context.logger = debugLogger('terafoundation-tests'); + }); + + afterAll(async () => { + await context.apis.foundation.promMetrics.shutdown(); + }); + + it('should initialize a promMetricsAPI', async () => { + const result = await context.apis.foundation.promMetrics.init(config); + expect(result).toBe(true); + }); + + it('should throw an error if promMetricsAPI is already initialized', async () => { + expect(() => context.apis.foundation.promMetrics.init(config)) + .toThrow('Prom metrics API cannot be initialized more than once.'); + }); }); }); - describe('in kubernetes clustering', () => { - describe('with prom metrics enabled in terafoundation', () => { - describe('with enable_prom_metrics undefined in jobConfig', () => { - const context = { - sysconfig: { - terafoundation: { - log_level: 'debug', - prom_metrics_enabled: true, - prom_metrics_port: 3333, - prom_metrics_add_default: true - }, - teraslice: { - cluster_manager_type: 'kubernetes' - } + describe('with prom metrics disabled in terafoundation', () => { + describe('with enable_prom_metrics undefined in jobConfig', () => { + const context = { + sysconfig: { + terafoundation: { + log_level: 'debug', + prom_metrics_enabled: false, + prom_metrics_port: 3333, + prom_metrics_add_default: true + }, + teraslice: { + cluster_manager_type: 'kubernetes', } - } as any; - - const config = { - context, - logger: debugLogger('prom-metrics-spec-logger'), - assignment: 'worker' - }; - - beforeAll(() => { - // This sets up the API endpoints in the context. - api(context); - context.logger = debugLogger('terafoundation-tests'); - }); - - afterAll(async () => { - await context.apis.foundation.promMetrics.shutdown(); - }); - - it('should initialize a promMetricsAPI', async () => { - const result = await context.apis.foundation.promMetrics.init(config); - expect(result).toBe(true); - }); - - it('should be able to verifyApi', async () => { - const apiExists = await context.apis.foundation.promMetrics.verifyAPI(); - expect(apiExists).toBe(true); - }); - - it('should throw an error if promMetricsAPI is already initialized', async () => { - expect(() => context.apis.foundation.promMetrics.init(config)) - .toThrow('Prom metrics API cannot be initialized more than once.'); - }); + }, + name: 'tera-test' + } as any; + + const { terafoundation } = context.sysconfig; + const config = { + tf_prom_metrics_enabled: terafoundation.prom_metrics_enabled, + tf_prom_metrics_port: terafoundation.prom_metrics_port, + tf_prom_metrics_add_default: terafoundation.prom_metrics_add_default, + logger: debugLogger('prom-metrics-spec-logger'), + assignment: 'worker' + }; + + beforeAll(() => { + // This sets up the API endpoints in the context. + api(context); + context.logger = debugLogger('terafoundation-tests'); }); - describe('with prom metrics disabled in jobConfig', () => { - const context = { - sysconfig: { - terafoundation: { - log_level: 'debug', - prom_metrics_enabled: true, - prom_metrics_port: 3333, - prom_metrics_add_default: true - }, - teraslice: { - cluster_manager_type: 'kubernetes' - } - } - } as any; - - const config = { - context, - logger: debugLogger('prom-metrics-spec-logger'), - assignment: 'worker', - metrics_enabled_by_job: false - }; - - beforeAll(() => { - // This sets up the API endpoints in the context. - api(context); - context.logger = debugLogger('terafoundation-tests'); - }); - - afterAll(async () => { - await context.apis.foundation.promMetrics.shutdown(); - }); - - it('should not initialize a promMetricsAPI', async () => { - const result = await context.apis.foundation.promMetrics.init(config); - expect(result).toBe(false); - }); + afterAll(async () => { + await context.apis.foundation.promMetrics.shutdown(); }); - describe('with prom metrics enabled in jobConfig', () => { - const context = { - sysconfig: { - terafoundation: { - log_level: 'debug', - prom_metrics_enabled: true, - prom_metrics_port: 3333, - prom_metrics_add_default: true - }, - teraslice: { - cluster_manager_type: 'kubernetes' - } - } - } as any; - - const config = { - context, - logger: debugLogger('prom-metrics-spec-logger'), - assignment: 'worker', - metrics_enabled_by_job: true - }; - - beforeAll(() => { - // This sets up the API endpoints in the context. - api(context); - context.logger = debugLogger('terafoundation-tests'); - }); - - afterAll(async () => { - await context.apis.foundation.promMetrics.shutdown(); - }); - - it('should initialize a promMetricsAPI', async () => { - const result = await context.apis.foundation.promMetrics.init(config); - expect(result).toBe(true); - }); - - it('should throw an error if promMetricsAPI is already initialized', async () => { - expect(() => context.apis.foundation.promMetrics.init(config)) - .toThrow('Prom metrics API cannot be initialized more than once.'); - }); + it('should not initialize a promMetricsAPI', async () => { + const result = await context.apis.foundation.promMetrics.init(config); + expect(result).toBe(false); }); }); - describe('with prom metrics disabled in terafoundation', () => { - describe('with enable_prom_metrics undefined in jobConfig', () => { - const context = { - sysconfig: { - terafoundation: { - log_level: 'debug', - prom_metrics_enabled: false, - prom_metrics_port: 3333, - prom_metrics_add_default: true - }, - teraslice: { - cluster_manager_type: 'kubernetes' - } + describe('with prom metrics disabled in jobConfig', () => { + const context = { + sysconfig: { + terafoundation: { + log_level: 'debug', + prom_metrics_enabled: false, + prom_metrics_port: 3333, + prom_metrics_add_default: true + }, + teraslice: { + cluster_manager_type: 'kubernetes', } - } as any; - - const config = { - context, - logger: debugLogger('prom-metrics-spec-logger'), - assignment: 'worker' - }; - - beforeAll(() => { - // This sets up the API endpoints in the context. - api(context); - context.logger = debugLogger('terafoundation-tests'); - }); - - afterAll(async () => { - await context.apis.foundation.promMetrics.shutdown(); - }); - - it('should not initialize a promMetricsAPI', async () => { - const result = await context.apis.foundation.promMetrics.init(config); - expect(result).toBe(false); - }); + }, + name: 'tera-test' + } as any; + + const { terafoundation } = context.sysconfig; + const config = { + tf_prom_metrics_enabled: terafoundation.prom_metrics_enabled, + tf_prom_metrics_port: terafoundation.prom_metrics_port, + tf_prom_metrics_add_default: terafoundation.prom_metrics_add_default, + logger: debugLogger('prom-metrics-spec-logger'), + assignment: 'worker', + job_prom_metrics_enabled: false + }; + + beforeAll(() => { + // This sets up the API endpoints in the context. + api(context); + context.logger = debugLogger('terafoundation-tests'); }); - describe('with prom metrics disabled in jobConfig', () => { - const context = { - sysconfig: { - terafoundation: { - log_level: 'debug', - prom_metrics_enabled: false, - prom_metrics_port: 3333, - prom_metrics_add_default: true - }, - teraslice: { - cluster_manager_type: 'kubernetes' - } - } - } as any; - - const config = { - context, - logger: debugLogger('prom-metrics-spec-logger'), - assignment: 'worker', - metrics_enabled_by_job: false - }; - - beforeAll(() => { - // This sets up the API endpoints in the context. - api(context); - context.logger = debugLogger('terafoundation-tests'); - }); - - afterAll(async () => { - await context.apis.foundation.promMetrics.shutdown(); - }); - - it('should not initialize a promMetricsAPI', async () => { - const result = await context.apis.foundation.promMetrics.init(config); - expect(result).toBe(false); - }); + afterAll(async () => { + await context.apis.foundation.promMetrics.shutdown(); }); - describe('with prom metrics enabled in jobConfig', () => { - const context = { - sysconfig: { - terafoundation: { - log_level: 'debug', - prom_metrics_enabled: false, - prom_metrics_port: 3333, - prom_metrics_add_default: true - }, - teraslice: { - cluster_manager_type: 'kubernetes' - } + it('should not initialize a promMetricsAPI', async () => { + const result = await context.apis.foundation.promMetrics.init(config); + expect(result).toBe(false); + }); + }); + + describe('with prom metrics enabled in jobConfig', () => { + const context = { + sysconfig: { + terafoundation: { + log_level: 'debug', + prom_metrics_enabled: false, + prom_metrics_port: 3333, + prom_metrics_add_default: true + }, + teraslice: { + cluster_manager_type: 'kubernetes', } - } as any; - - const config = { - context, - logger: debugLogger('prom-metrics-spec-logger'), - assignment: 'worker', - metrics_enabled_by_job: true - }; - - beforeAll(() => { - // This sets up the API endpoints in the context. - api(context); - context.logger = debugLogger('terafoundation-tests'); - }); - - afterAll(async () => { - await context.apis.foundation.promMetrics.shutdown(); - }); - - it('should initialize a promMetricsAPI', async () => { - const result = await context.apis.foundation.promMetrics.init(config); - expect(result).toBe(true); - }); - - it('should throw an error if promMetricsAPI is already initialized', async () => { - expect(() => context.apis.foundation.promMetrics.init(config)) - .toThrow('Prom metrics API cannot be initialized more than once.'); - }); + }, + name: 'tera-test' + } as any; + + const { terafoundation } = context.sysconfig; + const config = { + tf_prom_metrics_enabled: terafoundation.prom_metrics_enabled, + tf_prom_metrics_port: terafoundation.prom_metrics_port, + tf_prom_metrics_add_default: terafoundation.prom_metrics_add_default, + logger: debugLogger('prom-metrics-spec-logger'), + assignment: 'worker', + job_prom_metrics_enabled: true + }; + + beforeAll(() => { + // This sets up the API endpoints in the context. + api(context); + context.logger = debugLogger('terafoundation-tests'); + }); + + afterAll(async () => { + await context.apis.foundation.promMetrics.shutdown(); + }); + + it('should initialize a promMetricsAPI', async () => { + const result = await context.apis.foundation.promMetrics.init(config); + expect(result).toBe(true); + }); + + it('should throw an error if promMetricsAPI is already initialized', async () => { + expect(() => context.apis.foundation.promMetrics.init(config)) + .toThrow('Prom metrics API cannot be initialized more than once.'); }); }); }); @@ -311,17 +294,19 @@ describe('promMetrics foundation API', () => { prom_metrics_add_default: false }, teraslice: { - cluster_manager_type: 'kubernetes' + cluster_manager_type: 'kubernetes', } - } + }, + name: 'tera-test' } as any; + const { terafoundation } = context.sysconfig; const config = { - context, + tf_prom_metrics_enabled: terafoundation.prom_metrics_enabled, + tf_prom_metrics_port: terafoundation.prom_metrics_port, + tf_prom_metrics_add_default: terafoundation.prom_metrics_add_default, logger: debugLogger('prom-metrics-spec-logger'), assignment: 'cluster-master', - port: 3333, - default_metrics: false, labels: {}, prefix: 'foundation_test_' }; @@ -337,14 +322,17 @@ describe('promMetrics foundation API', () => { }); it('should be able to add a counter', async () => { - await context.apis.foundation.promMetrics.addMetric('counter1', 'help message', ['uuid'], 'counter'); + await context.apis.foundation.promMetrics.addCounter('counter1', 'help message', ['uuid'], function collect(this: Counter) { + const defaultLabels = context.apis.foundation.promMetrics.getDefaultLabels(); + this.inc({ uuid: '5g3kJr', ...defaultLabels }, 34); + }); const result = context.apis.foundation.promMetrics.hasMetric('counter1'); expect(result).toBe(true); }); it('should be able to increment a counter', async () => { context.apis.foundation.promMetrics.inc('counter1', { uuid: '5g3kJr' }, 17); - const response: Record = await got(`http://127.0.0.1:${config.port}/metrics`, { + const response: Record = await got(`http://127.0.0.1:${config.tf_prom_metrics_port}/metrics`, { throwHttpErrors: true }); @@ -353,7 +341,7 @@ describe('promMetrics foundation API', () => { .filter((line: string) => line.includes('5g3kJr'))[0] .split(' ')[1]; - expect(value).toBe('17'); + expect(value).toBe('51'); }); it('should be able to delete a counter', async () => { @@ -372,17 +360,19 @@ describe('promMetrics foundation API', () => { prom_metrics_add_default: true }, teraslice: { - cluster_manager_type: 'kubernetes' + cluster_manager_type: 'kubernetes', } - } + }, + name: 'tera-test' } as any; + const { terafoundation } = context.sysconfig; const config = { - context, + tf_prom_metrics_enabled: terafoundation.prom_metrics_enabled, + tf_prom_metrics_port: terafoundation.prom_metrics_port, + tf_prom_metrics_add_default: terafoundation.prom_metrics_add_default, logger: debugLogger('prom-metrics-spec-logger'), assignment: 'cluster-master', - port: 3333, - default_metrics: true, labels: {}, prefix: 'foundation_test_' }; @@ -398,14 +388,17 @@ describe('promMetrics foundation API', () => { }); it('should be able to add a gauge', async () => { - await context.apis.foundation.promMetrics.addMetric('gauge1', 'help message', ['uuid'], 'gauge'); + await context.apis.foundation.promMetrics.addGauge('gauge1', 'help message', ['uuid'], function collect(this: Gauge) { + const defaultLabels = context.apis.foundation.promMetrics.getDefaultLabels(); + this.inc({ uuid: 'h3L8JB6i', ...defaultLabels }, 1); + }); const result = context.apis.foundation.promMetrics.hasMetric('gauge1'); expect(result).toBe(true); }); it('should be able to increment a gauge', async () => { context.apis.foundation.promMetrics.inc('gauge1', { uuid: 'h3L8JB6i' }, 28); - const response: Record = await got(`http://127.0.0.1:${config.port}/metrics`, { + const response: Record = await got(`http://127.0.0.1:${config.tf_prom_metrics_port}/metrics`, { throwHttpErrors: true }); @@ -414,12 +407,12 @@ describe('promMetrics foundation API', () => { .filter((line: string) => line.includes('h3L8JB6i'))[0] .split(' ')[1]; - expect(value).toBe('28'); + expect(value).toBe('29'); }); it('should be able to decrement a gauge', async () => { - context.apis.foundation.promMetrics.dec('gauge1', { uuid: 'h3L8JB6i' }, 1); - const response: Record = await got(`http://127.0.0.1:${config.port}/metrics`, { + context.apis.foundation.promMetrics.dec('gauge1', { uuid: 'h3L8JB6i' }, 5); + const response: Record = await got(`http://127.0.0.1:${config.tf_prom_metrics_port}/metrics`, { throwHttpErrors: true }); @@ -428,12 +421,12 @@ describe('promMetrics foundation API', () => { .filter((line: string) => line.includes('h3L8JB6i'))[0] .split(' ')[1]; - expect(value).toBe('27'); + expect(value).toBe('25'); }); it('should be able to set a gauge', async () => { context.apis.foundation.promMetrics.set('gauge1', { uuid: 'h3L8JB6i' }, 103); - const response: Record = await got(`http://127.0.0.1:${config.port}/metrics`, { + const response: Record = await got(`http://127.0.0.1:${config.tf_prom_metrics_port}/metrics`, { throwHttpErrors: true }); @@ -442,7 +435,7 @@ describe('promMetrics foundation API', () => { .filter((line: string) => line.includes('h3L8JB6i'))[0] .split(' ')[1]; - expect(value).toBe('103'); + expect(value).toBe('104'); }); it('should be able to delete a gauge', async () => { @@ -461,13 +454,17 @@ describe('promMetrics foundation API', () => { prom_metrics_add_default: false }, teraslice: { - cluster_manager_type: 'kubernetes' + cluster_manager_type: 'kubernetes', } - } + }, + name: 'tera-test' } as any; + const { terafoundation } = context.sysconfig; const config = { - context, + tf_prom_metrics_enabled: terafoundation.prom_metrics_enabled, + tf_prom_metrics_port: terafoundation.prom_metrics_port, + tf_prom_metrics_add_default: terafoundation.prom_metrics_add_default, logger: debugLogger('prom-metrics-spec-logger'), assignment: 'cluster-master', prefix: 'foundation_test_' @@ -485,12 +482,15 @@ describe('promMetrics foundation API', () => { }); it('should be able to add a histogram', async () => { - await context.apis.foundation.promMetrics.addMetric( + await context.apis.foundation.promMetrics.addHistogram( 'histogram1', 'help message', ['uuid'], - 'histogram', - [0.1, 5, 15, 50, 100, 500] + function collect(this: Histogram) { + const defaultLabels = context.apis.foundation.promMetrics.getDefaultLabels(); + this.observe({ uuid: '5Mw4Zfx2', ...defaultLabels }, 1000); + }, + [0.1, 5, 15, 50, 100, 500, 1000] ); const result = context.apis.foundation.promMetrics.hasMetric('histogram1'); expect(result).toBe(true); @@ -512,10 +512,10 @@ describe('promMetrics foundation API', () => { expect(values .filter((line: string) => line.includes('histogram1_sum'))[0] - .split(' ')[1]).toBe('170'); + .split(' ')[1]).toBe('1170'); expect(values .filter((line: string) => line.includes('histogram1_count'))[0] - .split(' ')[1]).toBe('5'); + .split(' ')[1]).toBe('6'); }); it('should be able to delete a histogram', async () => { @@ -534,13 +534,17 @@ describe('promMetrics foundation API', () => { prom_metrics_add_default: false }, teraslice: { - cluster_manager_type: 'kubernetes' + cluster_manager_type: 'kubernetes', } - } + }, + name: 'tera-test' } as any; + const { terafoundation } = context.sysconfig; const config = { - context, + tf_prom_metrics_enabled: terafoundation.prom_metrics_enabled, + tf_prom_metrics_port: terafoundation.prom_metrics_port, + tf_prom_metrics_add_default: terafoundation.prom_metrics_add_default, logger: debugLogger('prom-metrics-spec-logger'), assignment: 'cluster-master', prefix: 'foundation_test_' @@ -561,6 +565,10 @@ describe('promMetrics foundation API', () => { 'summary1', 'help message', ['uuid'], + function collect(this: Summary) { + const defaultLabels = context.apis.foundation.promMetrics.getDefaultLabels(); + this.observe({ uuid: 'nHy34Ol9', ...defaultLabels }, 1000); + }, 60, 5, [0.1, 0.5, 0.9] @@ -583,10 +591,10 @@ describe('promMetrics foundation API', () => { expect(values .filter((line: string) => line.includes('summary1_sum'))[0] - .split(' ')[1]).toBe('33'); + .split(' ')[1]).toBe('1033'); expect(values .filter((line: string) => line.includes('summary1_count'))[0] - .split(' ')[1]).toBe('3'); + .split(' ')[1]).toBe('4'); }); it('should be able to delete a summary', async () => { @@ -594,4 +602,49 @@ describe('promMetrics foundation API', () => { expect(context.apis.foundation.promMetrics.hasMetric('summary1')).toBe(false); }); }); + + describe('getDefaultLabels', () => { + const context = { + sysconfig: { + terafoundation: { + log_level: 'debug', + prom_metrics_enabled: true, + prom_metrics_port: 3337, + prom_metrics_add_default: false + }, + teraslice: { + cluster_manager_type: 'kubernetes', + } + }, + name: 'tera-test-labels' + } as any; + + const { terafoundation } = context.sysconfig; + const config = { + tf_prom_metrics_enabled: terafoundation.prom_metrics_enabled, + tf_prom_metrics_port: terafoundation.prom_metrics_port, + tf_prom_metrics_add_default: terafoundation.prom_metrics_add_default, + logger: debugLogger('prom-metrics-spec-logger'), + assignment: 'cluster-master', + prefix: 'foundation_test_', + labels: { default1: 'value1' } + }; + beforeAll(async () => { + // This sets up the API endpoints in the context. + api(context); + context.logger = debugLogger('terafoundation-tests'); + await context.apis.foundation.promMetrics.init(config); + }); + + afterAll(async () => { + await context.apis.foundation.promMetrics.shutdown(); + }); + it('should get all the default labels', () => { + expect(context.apis.foundation.promMetrics.getDefaultLabels()).toEqual({ + name: 'tera-test-labels', + assignment: 'cluster-master', + default1: 'value1' + }); + }); + }); }); diff --git a/packages/terafoundation/test/test-context-spec.ts b/packages/terafoundation/test/test-context-spec.ts index b87a271d9c6..a01f0f25ff9 100644 --- a/packages/terafoundation/test/test-context-spec.ts +++ b/packages/terafoundation/test/test-context-spec.ts @@ -126,7 +126,9 @@ describe('TestContext', () => { context.sysconfig.teraslice = { cluster_manager_type: 'kubernetes' }; context.sysconfig.terafoundation.prom_metrics_enabled = true; const config = { - context, + tf_prom_metrics_enabled: true, + tf_prom_metrics_port: 3333, + tf_prom_metrics_add_default: false, logger: context.logger, assignment: 'cluster-master' }; diff --git a/packages/teraslice-cli/package.json b/packages/teraslice-cli/package.json index 15380b2d814..f802e97abd4 100644 --- a/packages/teraslice-cli/package.json +++ b/packages/teraslice-cli/package.json @@ -1,7 +1,7 @@ { "name": "teraslice-cli", "displayName": "Teraslice CLI", - "version": "1.3.1", + "version": "1.4.0", "description": "Command line manager for teraslice jobs, assets, and cluster references.", "keywords": [ "teraslice" @@ -38,8 +38,8 @@ }, "dependencies": { "@terascope/fetch-github-release": "^0.8.7", - "@terascope/types": "^0.16.0", - "@terascope/utils": "^0.58.1", + "@terascope/types": "^0.17.0", + "@terascope/utils": "^0.59.0", "chalk": "^4.1.2", "cli-table3": "^0.6.4", "easy-table": "^1.2.0", @@ -54,7 +54,7 @@ "pretty-bytes": "^5.6.0", "prompts": "^2.4.2", "signale": "^1.4.0", - "teraslice-client-js": "^0.59.1", + "teraslice-client-js": "^0.60.0", "tmp": "^0.2.0", "tty-table": "^4.2.3", "yargs": "^17.7.2", diff --git a/packages/teraslice-client-js/package.json b/packages/teraslice-client-js/package.json index b5817ffb9c0..559bf2d4a0e 100644 --- a/packages/teraslice-client-js/package.json +++ b/packages/teraslice-client-js/package.json @@ -1,7 +1,7 @@ { "name": "teraslice-client-js", "displayName": "Teraslice Client (JavaScript)", - "version": "0.59.1", + "version": "0.60.0", "description": "A Node.js client for teraslice jobs, assets, and cluster references.", "keywords": [ "elasticsearch", @@ -31,8 +31,8 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/types": "^0.16.0", - "@terascope/utils": "^0.58.1", + "@terascope/types": "^0.17.0", + "@terascope/utils": "^0.59.0", "auto-bind": "^4.0.0", "got": "^11.8.3" }, diff --git a/packages/teraslice-messaging/package.json b/packages/teraslice-messaging/package.json index dfddf3dd35a..50b3a6572e8 100644 --- a/packages/teraslice-messaging/package.json +++ b/packages/teraslice-messaging/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/teraslice-messaging", "displayName": "Teraslice Messaging", - "version": "0.41.1", + "version": "0.42.0", "description": "An internal teraslice messaging library using socket.io", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/teraslice-messaging#readme", "bugs": { @@ -34,8 +34,8 @@ "ms": "^2.1.3" }, "dependencies": { - "@terascope/types": "^0.16.0", - "@terascope/utils": "^0.58.1", + "@terascope/types": "^0.17.0", + "@terascope/utils": "^0.59.0", "ms": "^2.1.3", "nanoid": "^3.3.4", "p-event": "^4.2.0", diff --git a/packages/teraslice-op-test-harness/package.json b/packages/teraslice-op-test-harness/package.json index 0caf5198129..d9935c3b81f 100644 --- a/packages/teraslice-op-test-harness/package.json +++ b/packages/teraslice-op-test-harness/package.json @@ -21,10 +21,10 @@ "bluebird": "^3.7.2" }, "devDependencies": { - "@terascope/job-components": "^0.73.2" + "@terascope/job-components": "^0.74.0" }, "peerDependencies": { - "@terascope/job-components": ">=0.73.2" + "@terascope/job-components": ">=0.74.0" }, "engines": { "node": ">=14.17.0", diff --git a/packages/teraslice-state-storage/package.json b/packages/teraslice-state-storage/package.json index 9b7914dd6bc..255f98dc97f 100644 --- a/packages/teraslice-state-storage/package.json +++ b/packages/teraslice-state-storage/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/teraslice-state-storage", "displayName": "Teraslice State Storage", - "version": "0.52.1", + "version": "0.53.0", "description": "State storage operation api for teraslice", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/teraslice-state-storage#readme", "bugs": { @@ -23,8 +23,8 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/elasticsearch-api": "^3.19.1", - "@terascope/utils": "^0.58.1" + "@terascope/elasticsearch-api": "^3.20.0", + "@terascope/utils": "^0.59.0" }, "engines": { "node": ">=14.17.0", diff --git a/packages/teraslice-test-harness/package.json b/packages/teraslice-test-harness/package.json index e6f768724dd..9bdfb043ec7 100644 --- a/packages/teraslice-test-harness/package.json +++ b/packages/teraslice-test-harness/package.json @@ -36,10 +36,10 @@ "fs-extra": "^11.2.0" }, "devDependencies": { - "@terascope/job-components": "^0.73.2" + "@terascope/job-components": "^0.74.0" }, "peerDependencies": { - "@terascope/job-components": ">=0.73.2" + "@terascope/job-components": ">=0.74.0" }, "engines": { "node": ">=14.17.0", diff --git a/packages/teraslice/package.json b/packages/teraslice/package.json index ac9cd9854ca..0fd311df240 100644 --- a/packages/teraslice/package.json +++ b/packages/teraslice/package.json @@ -1,7 +1,7 @@ { "name": "teraslice", "displayName": "Teraslice", - "version": "1.4.1", + "version": "1.5.0", "description": "Distributed computing platform for processing JSON data", "homepage": "https://github.com/terascope/teraslice#readme", "bugs": { @@ -38,11 +38,11 @@ "ms": "^2.1.3" }, "dependencies": { - "@terascope/elasticsearch-api": "^3.19.1", - "@terascope/job-components": "^0.73.2", - "@terascope/teraslice-messaging": "^0.41.1", - "@terascope/types": "^0.16.0", - "@terascope/utils": "^0.58.1", + "@terascope/elasticsearch-api": "^3.20.0", + "@terascope/job-components": "^0.74.0", + "@terascope/teraslice-messaging": "^0.42.0", + "@terascope/types": "^0.17.0", + "@terascope/utils": "^0.59.0", "async-mutex": "^0.5.0", "barbe": "^3.0.16", "body-parser": "^1.20.2", @@ -64,7 +64,7 @@ "semver": "^7.6.1", "socket.io": "^1.7.4", "socket.io-client": "^1.7.4", - "terafoundation": "^0.62.1", + "terafoundation": "^0.63.0", "uuid": "^9.0.1" }, "devDependencies": { diff --git a/packages/teraslice/src/lib/cluster/cluster_master.ts b/packages/teraslice/src/lib/cluster/cluster_master.ts index 31da566f3bb..56272dbae9e 100644 --- a/packages/teraslice/src/lib/cluster/cluster_master.ts +++ b/packages/teraslice/src/lib/cluster/cluster_master.ts @@ -141,33 +141,20 @@ export class ClusterMaster { await services.apiService.initialize(); /// initialize promClient - await this.context.apis.foundation.promMetrics.init({ - context: this.context, - logger: this.logger, - assignment: 'cluster_master', - port: this.context.sysconfig.terafoundation.prom_metrics_port, - }); - - await this.setupPromMetrics(); - - await this.context.apis.foundation.promMetrics.addMetric( - 'info', - 'Information about Teraslice cluster master', - ['arch', 'clustering_type', 'name', 'node_version', 'platform', 'teraslice_version'], - 'gauge' - ); - this.context.apis.foundation.promMetrics.set( - 'info', - { - arch: this.context.arch, - clustering_type: this.context.sysconfig.teraslice.cluster_manager_type, - name: this.context.sysconfig.teraslice.name, - node_version: process.version, - platform: this.context.platform, - teraslice_version: getPackageJSON().version - }, - 1 - ); + if (this.context.sysconfig.teraslice.cluster_manager_type === 'native') { + this.logger.warn('Skipping PromMetricsAPI initialization: incompatible with native clustering.'); + } else { + const { terafoundation } = this.context.sysconfig; + await this.context.apis.foundation.promMetrics.init({ + tf_prom_metrics_add_default: terafoundation.prom_metrics_add_default, + tf_prom_metrics_enabled: terafoundation.prom_metrics_enabled, + tf_prom_metrics_port: terafoundation.prom_metrics_port, + logger: this.logger, + assignment: 'cluster_master' + }); + + await this.setupPromMetrics(); + } this.logger.info('cluster master is ready!'); this.running = true; @@ -241,78 +228,71 @@ export class ClusterMaster { */ await Promise.all([ - this.context.apis.foundation.promMetrics.addMetric( + this.context.apis.foundation.promMetrics.addGauge( + 'info', + 'Information about Teraslice cluster master', + ['arch', 'clustering_type', 'name', 'node_version', 'platform', 'teraslice_version'] + ), + this.context.apis.foundation.promMetrics.addGauge( 'controller_workers_active', 'Number of Teraslice workers actively processing slices.', ['ex_id', 'job_id', 'job_name'], - 'gauge' ), - this.context.apis.foundation.promMetrics.addMetric( + this.context.apis.foundation.promMetrics.addGauge( 'controller_workers_available', 'Number of Teraslice workers running and waiting for work.', ['ex_id', 'job_id', 'job_name'], - 'gauge' ), - this.context.apis.foundation.promMetrics.addMetric( + this.context.apis.foundation.promMetrics.addGauge( 'controller_workers_joined', 'Total number of Teraslice workers that have joined the execution controller for this job.', ['ex_id', 'job_id', 'job_name'], - 'gauge' ), - this.context.apis.foundation.promMetrics.addMetric( + this.context.apis.foundation.promMetrics.addGauge( 'controller_workers_reconnected', 'Total number of Teraslice workers that have reconnected to the execution controller for this job.', ['ex_id', 'job_id', 'job_name'], - 'gauge' ), - this.context.apis.foundation.promMetrics.addMetric( + this.context.apis.foundation.promMetrics.addGauge( 'controller_workers_disconnected', 'Total number of Teraslice workers that have disconnected from execution controller for this job.', ['ex_id', 'job_id', 'job_name'], - 'gauge' ), - this.context.apis.foundation.promMetrics.addMetric( + this.context.apis.foundation.promMetrics.addGauge( 'execution_info', 'Information about Teraslice execution.', ['ex_id', 'job_id', 'image', 'version'], - 'gauge' ), - this.context.apis.foundation.promMetrics.addMetric( + this.context.apis.foundation.promMetrics.addGauge( 'controller_slicers_count', 'Number of execution controllers (slicers) running for this execution.', ['ex_id', 'job_id', 'job_name'], - 'gauge' ), // Execution Related Metrics - this.context.apis.foundation.promMetrics.addMetric( + this.context.apis.foundation.promMetrics.addGauge( 'execution_cpu_limit', 'CPU core limit for a Teraslice worker container.', ['ex_id', 'job_id', 'job_name'], - 'gauge' ), - this.context.apis.foundation.promMetrics.addMetric( + this.context.apis.foundation.promMetrics.addGauge( 'execution_cpu_request', 'Requested number of CPU cores for a Teraslice worker container.', ['ex_id', 'job_id', 'job_name'], - 'gauge' ), - this.context.apis.foundation.promMetrics.addMetric( + this.context.apis.foundation.promMetrics.addGauge( 'execution_memory_limit', 'Memory limit for Teraslice a worker container.', ['ex_id', 'job_id', 'job_name'], - 'gauge' ), - this.context.apis.foundation.promMetrics.addMetric( + this.context.apis.foundation.promMetrics.addGauge( 'execution_memory_request', 'Requested amount of memory for a Teraslice worker container.', ['ex_id', 'job_id', 'job_name'], - 'gauge' ), - this.context.apis.foundation.promMetrics.addMetric( + this.context.apis.foundation.promMetrics.addGauge( 'execution_status', 'Current status of the Teraslice execution.', ['ex_id', 'job_id', 'job_name', 'status'], - 'gauge' ), /* TODO: The following gauges should be Counters. This was not done because @@ -320,48 +300,54 @@ export class ClusterMaster { So setting the gauge is the only real way to gather the metrics in master. Solution to convert would be setting the count in the ex process. */ - this.context.apis.foundation.promMetrics.addMetric( + this.context.apis.foundation.promMetrics.addGauge( 'controller_slices_processed', 'Number of slices processed.', ['ex_id', 'job_id', 'job_name'], - 'gauge' ), - this.context.apis.foundation.promMetrics.addMetric( + this.context.apis.foundation.promMetrics.addGauge( 'controller_slices_failed', 'Number of slices failed.', ['ex_id', 'job_id', 'job_name'], - 'gauge' ), - this.context.apis.foundation.promMetrics.addMetric( + this.context.apis.foundation.promMetrics.addGauge( 'controller_slices_queued', 'Number of slices queued for processing.', ['ex_id', 'job_id', 'job_name'], - 'gauge' ), - this.context.apis.foundation.promMetrics.addMetric( + this.context.apis.foundation.promMetrics.addGauge( 'execution_created_timestamp_seconds', 'Execution creation time.', ['ex_id', 'job_id', 'job_name'], - 'gauge' ), - this.context.apis.foundation.promMetrics.addMetric( + this.context.apis.foundation.promMetrics.addGauge( 'execution_updated_timestamp_seconds', 'Execution update time.', ['ex_id', 'job_id', 'job_name'], - 'gauge' ), - this.context.apis.foundation.promMetrics.addMetric( + this.context.apis.foundation.promMetrics.addGauge( 'execution_slicers', 'Number of slicers defined on the execution.', ['ex_id', 'job_id', 'job_name'], - 'gauge' ), - this.context.apis.foundation.promMetrics.addMetric( + this.context.apis.foundation.promMetrics.addGauge( 'execution_workers', 'Number of workers defined on the execution. Note that the number of actual workers can differ from this value.', ['ex_id', 'job_id', 'job_name'], - 'gauge' ), ]); + + this.context.apis.foundation.promMetrics.set( + 'info', + { + arch: this.context.arch, + clustering_type: this.context.sysconfig.teraslice.cluster_manager_type, + name: this.context.sysconfig.teraslice.name, + node_version: process.version, + platform: this.context.platform, + teraslice_version: getPackageJSON().version + }, + 1 + ); } } diff --git a/packages/teraslice/src/lib/workers/execution-controller/index.ts b/packages/teraslice/src/lib/workers/execution-controller/index.ts index 2aab6e7cadc..d79c28eb3fb 100644 --- a/packages/teraslice/src/lib/workers/execution-controller/index.ts +++ b/packages/teraslice/src/lib/workers/execution-controller/index.ts @@ -136,6 +136,28 @@ export class ExecutionController { } async initialize() { + if (this.context.sysconfig.teraslice.cluster_manager_type === 'native') { + this.logger.warn('Skipping PromMetricsAPI initialization: incompatible with native clustering.'); + } else { + const { terafoundation } = this.context.sysconfig; + await this.context.apis.foundation.promMetrics.init({ + assignment: 'execution_controller', + logger: this.logger, + tf_prom_metrics_add_default: terafoundation.prom_metrics_add_default, + tf_prom_metrics_enabled: terafoundation.prom_metrics_enabled, + tf_prom_metrics_port: terafoundation.prom_metrics_port, + job_prom_metrics_add_default: this.executionContext.config.prom_metrics_add_default, + job_prom_metrics_enabled: this.executionContext.config.prom_metrics_enabled, + job_prom_metrics_port: this.executionContext.config.prom_metrics_port, + labels: { + ex_id: this.exId, + job_id: this.executionContext.jobId, + job_name: this.executionContext.config.name, + } + }); + } + await this.setupPromMetrics(); + await Promise.all([ this.executionStorage.initialize(), this.stateStorage.initialize(), @@ -1080,4 +1102,51 @@ export class ExecutionController { this.pendingDispatches++; } + + /** + * Adds all prom metrics specific to the execution_controller. + * + * If trying to add a new metric for the execution_controller, it belongs here. + * @async + * @function setupPromMetrics + * @return {Promise} + * @link https://terascope.github.io/teraslice/docs/development/k8s#prometheus-metrics-api + */ + async setupPromMetrics() { + this.logger.info(`adding ${this.context.assignment} prom metrics...`); + // eslint-disable-next-line @typescript-eslint/no-this-alias + const self = this; + await Promise.all([ + this.context.apis.foundation.promMetrics.addGauge( + 'info', + 'Information about Teraslice execution controller', + ['arch', 'clustering_type', 'name', 'node_version', 'platform', 'teraslice_version'], + ), + this.context.apis.foundation.promMetrics.addGauge( + 'slices_processed', + 'Number of slices processed by all workers', + [], + function collect() { + const slicesProcessed = self.executionAnalytics.get('processed'); + const defaultLabels = { + ...self.context.apis.foundation.promMetrics.getDefaultLabels() + }; + this.set(defaultLabels, slicesProcessed); + } + ) + ]); + + this.context.apis.foundation.promMetrics.set( + 'info', + { + arch: this.context.arch, + clustering_type: this.context.sysconfig.teraslice.cluster_manager_type, + name: this.context.sysconfig.teraslice.name, + node_version: process.version, + platform: this.context.platform, + teraslice_version: this.executionContext.config.teraslice_version + }, + 1 + ); + } } diff --git a/packages/teraslice/src/lib/workers/worker/index.ts b/packages/teraslice/src/lib/workers/worker/index.ts index 208338c1352..d2dd4e55c94 100644 --- a/packages/teraslice/src/lib/workers/worker/index.ts +++ b/packages/teraslice/src/lib/workers/worker/index.ts @@ -85,6 +85,28 @@ export class Worker { } async initialize() { + if (this.context.sysconfig.teraslice.cluster_manager_type === 'native') { + this.logger.warn('Skipping PromMetricsAPI initialization: incompatible with native clustering.'); + } else { + const { terafoundation } = this.context.sysconfig; + await this.context.apis.foundation.promMetrics.init({ + assignment: 'worker', + logger: this.logger, + tf_prom_metrics_add_default: terafoundation.prom_metrics_add_default, + tf_prom_metrics_enabled: terafoundation.prom_metrics_enabled, + tf_prom_metrics_port: terafoundation.prom_metrics_port, + job_prom_metrics_add_default: this.executionContext.config.prom_metrics_add_default, + job_prom_metrics_enabled: this.executionContext.config.prom_metrics_enabled, + job_prom_metrics_port: this.executionContext.config.prom_metrics_port, + labels: { + ex_id: this.executionContext.exId, + job_id: this.executionContext.jobId, + job_name: this.executionContext.config.name, + } + }); + } + await this.setupPromMetrics(); + const { context } = this; this.isInitialized = true; @@ -373,4 +395,55 @@ export class Worker { }, this.shutdownTimeout); }); } + + getSlicesProcessed() { + return this.slicesProcessed; + } + + /** + * Adds all prom metrics specific to the worker. + * + * If trying to add a new metric for the worker, it belongs here. + * @async + * @function setupPromMetrics + * @return {Promise} + * @link https://terascope.github.io/teraslice/docs/development/k8s#prometheus-metrics-api + */ + async setupPromMetrics() { + this.logger.info(`adding ${this.context.assignment} prom metrics...`); + // eslint-disable-next-line @typescript-eslint/no-this-alias + const self = this; + await Promise.all([ + this.context.apis.foundation.promMetrics.addGauge( + 'info', + 'Information about Teraslice worker', + ['arch', 'clustering_type', 'name', 'node_version', 'platform', 'teraslice_version'], + ), + this.context.apis.foundation.promMetrics.addGauge( + 'slices_processed', + 'Number of slices the worker has processed', + [], + function collect() { + const slicesProcessed = self.getSlicesProcessed(); + const defaultLabels = { + ...self.context.apis.foundation.promMetrics.getDefaultLabels() + }; + this.set(defaultLabels, slicesProcessed); + } + ) + ]); + + this.context.apis.foundation.promMetrics.set( + 'info', + { + arch: this.context.arch, + clustering_type: this.context.sysconfig.teraslice.cluster_manager_type, + name: this.context.sysconfig.teraslice.name, + node_version: process.version, + platform: this.context.platform, + teraslice_version: this.executionContext.config.teraslice_version + }, + 1 + ); + } } diff --git a/packages/ts-transforms/package.json b/packages/ts-transforms/package.json index f2359000661..ad8ec07f4f0 100644 --- a/packages/ts-transforms/package.json +++ b/packages/ts-transforms/package.json @@ -1,7 +1,7 @@ { "name": "ts-transforms", "displayName": "TS Transforms", - "version": "0.84.1", + "version": "0.85.0", "description": "An ETL framework built upon xlucene-evaluator", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/ts-transforms#readme", "bugs": { @@ -35,9 +35,9 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/data-mate": "^0.55.1", - "@terascope/types": "^0.16.0", - "@terascope/utils": "^0.58.1", + "@terascope/data-mate": "^0.56.0", + "@terascope/types": "^0.17.0", + "@terascope/utils": "^0.59.0", "awesome-phonenumber": "^2.70.0", "graphlib": "^2.1.8", "is-ip": "^3.1.0", diff --git a/packages/types/package.json b/packages/types/package.json index e76bf28ee77..7fd872083a3 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/types", "displayName": "Types", - "version": "0.16.0", + "version": "0.17.0", "description": "A collection of typescript interfaces", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/types#readme", "bugs": { diff --git a/packages/types/src/terafoundation.ts b/packages/types/src/terafoundation.ts index fc9fc10fd28..f8210641e7a 100644 --- a/packages/types/src/terafoundation.ts +++ b/packages/types/src/terafoundation.ts @@ -3,6 +3,9 @@ import { Cluster as NodeJSCluster, Worker as NodeJSWorker } from 'node:cluster'; +import { + CollectFunction, Counter, Gauge, Histogram, Summary +} from 'prom-client'; import type { Overwrite } from './utility'; import type { Logger } from './logger'; @@ -168,13 +171,19 @@ export type Context< cluster: Cluster; } -export interface PromMetricsInitConfig extends Omit { - context: Context, +export interface PromMetricsInitConfig { + assignment: string logger: Logger, - metrics_enabled_by_job?: boolean, - port?: number - default_metrics?: boolean + tf_prom_metrics_enabled: boolean; + tf_prom_metrics_port: number; + tf_prom_metrics_add_default: boolean; + job_prom_metrics_enabled?: boolean, + job_prom_metrics_port?: number + job_prom_metrics_add_default?: boolean + labels?: Record, + prefix?: string } + export interface PromMetricsAPIConfig { assignment: string port: number @@ -189,13 +198,28 @@ export interface PromMetrics { inc: (name: string, labelValues: Record, value: number) => void; dec: (name: string, labelValues: Record, value: number) => void; observe: (name: string, labelValues: Record, value: number) => void; - addMetric: (name: string, help: string, labelNames: Array, type: 'gauge' | 'counter' | 'histogram', - buckets?: Array) => Promise; + addGauge: (name: string, help: string, labelNames: Array, + collectFn?: CollectFunction) => Promise; + addCounter: (name: string, help: string, labelNames: Array, + collectFn?: CollectFunction) => Promise; + addHistogram: (name: string, help: string, labelNames: Array, + collectFn?: CollectFunction, buckets?: Array) => Promise; addSummary: (name: string, help: string, labelNames: Array, - ageBuckets: number, maxAgeSeconds: number, - percentiles: Array) => Promise; + collectFn?: CollectFunction, maxAgeSeconds?: number, + ageBuckets?: number, percentiles?: Array) => Promise; hasMetric: (name: string) => boolean; deleteMetric: (name: string) => Promise; verifyAPI: () => boolean; shutdown: () => Promise; + getDefaultLabels: () => Record; } + +export type MetricList = Record | Counter | Histogram | Summary, + readonly functions?: Set +}>; + +export type { + CollectFunction, Counter, Gauge, Histogram, Summary +} from 'prom-client'; diff --git a/packages/utils/package.json b/packages/utils/package.json index bd49f56e36b..5aa605c3f5b 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/utils", "displayName": "Utils", - "version": "0.58.1", + "version": "0.59.0", "description": "A collection of Teraslice Utilities", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/utils#readme", "bugs": { @@ -28,7 +28,7 @@ "debug": "^4.3.4" }, "dependencies": { - "@terascope/types": "^0.16.0", + "@terascope/types": "^0.17.0", "@turf/bbox": "^6.4.0", "@turf/bbox-polygon": "^6.4.0", "@turf/boolean-contains": "^6.4.0", diff --git a/packages/xlucene-parser/package.json b/packages/xlucene-parser/package.json index c6c9ef25434..ba4c238db7a 100644 --- a/packages/xlucene-parser/package.json +++ b/packages/xlucene-parser/package.json @@ -1,7 +1,7 @@ { "name": "xlucene-parser", "displayName": "xLucene Parser", - "version": "0.57.1", + "version": "0.58.0", "description": "Flexible Lucene-like evaluator and language parser", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/xlucene-parser#readme", "repository": { @@ -30,8 +30,8 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/types": "^0.16.0", - "@terascope/utils": "^0.58.1" + "@terascope/types": "^0.17.0", + "@terascope/utils": "^0.59.0" }, "devDependencies": { "@turf/invariant": "^6.2.0", diff --git a/packages/xlucene-translator/package.json b/packages/xlucene-translator/package.json index aaebdfb8a82..b51017e0f23 100644 --- a/packages/xlucene-translator/package.json +++ b/packages/xlucene-translator/package.json @@ -1,7 +1,7 @@ { "name": "xlucene-translator", "displayName": "xLucene Translator", - "version": "0.43.1", + "version": "0.44.0", "description": "Translate xlucene query to database queries", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/xlucene-translator#readme", "repository": { @@ -28,10 +28,10 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/types": "^0.16.0", - "@terascope/utils": "^0.58.1", + "@terascope/types": "^0.17.0", + "@terascope/utils": "^0.59.0", "@types/elasticsearch": "^5.0.43", - "xlucene-parser": "^0.57.1" + "xlucene-parser": "^0.58.0" }, "devDependencies": { "elasticsearch": "^15.4.1" diff --git a/packages/xpressions/package.json b/packages/xpressions/package.json index f6291a6b0df..0f623116c03 100644 --- a/packages/xpressions/package.json +++ b/packages/xpressions/package.json @@ -1,7 +1,7 @@ { "name": "xpressions", "displayName": "Xpressions", - "version": "0.25.1", + "version": "0.26.0", "description": "Variable expressions with date-math support", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/xpressions#readme", "bugs": { @@ -23,10 +23,10 @@ "test:watch": "ts-scripts test --watch . --" }, "dependencies": { - "@terascope/utils": "^0.58.1" + "@terascope/utils": "^0.59.0" }, "devDependencies": { - "@terascope/types": "^0.16.0" + "@terascope/types": "^0.17.0" }, "engines": { "node": ">=14.17.0",