From c8bc86b90f88a37bbc6d4d1bede77093ccbcfede Mon Sep 17 00:00:00 2001 From: "nkl199@yahoo.co.uk" Date: Thu, 27 Aug 2020 11:49:38 +0100 Subject: [PATCH] Update push gateway to use prom-client and match metrics with prometheus scrape Signed-off-by: nkl199@yahoo.co.uk --- .../caliper-core/lib/common/config/Config.js | 6 + .../lib/common/config/default.yaml | 8 + .../prometheus/prometheus-push-client.js | 146 ----------------- .../lib/common/utils/caliper-utils.js | 29 +++- .../lib/common/utils/constants.js | 3 + .../prometheus-push-tx-observer.js | 147 +++++++++++++----- .../test/common/utils/caliper-utils.js | 62 ++++++++ 7 files changed, 210 insertions(+), 191 deletions(-) delete mode 100644 packages/caliper-core/lib/common/prometheus/prometheus-push-client.js create mode 100644 packages/caliper-core/test/common/utils/caliper-utils.js diff --git a/packages/caliper-core/lib/common/config/Config.js b/packages/caliper-core/lib/common/config/Config.js index b5fbdda1f9..5c888d63e8 100644 --- a/packages/caliper-core/lib/common/config/Config.js +++ b/packages/caliper-core/lib/common/config/Config.js @@ -21,6 +21,12 @@ const nconf = require('nconf'); nconf.formats.yaml = require('nconf-yaml'); const keys = { + Auth: { + PrometheusPush: { + UserName: 'caliper-auth-prometheuspush-username', + Password: 'caliper-auth-prometheuspush-password' + } + }, Bind: { Sut: 'caliper-bind-sut', Args: 'caliper-bind-args', diff --git a/packages/caliper-core/lib/common/config/default.yaml b/packages/caliper-core/lib/common/config/default.yaml index 572edcadce..ae01343672 100644 --- a/packages/caliper-core/lib/common/config/default.yaml +++ b/packages/caliper-core/lib/common/config/default.yaml @@ -13,6 +13,14 @@ # caliper: + # Settings related to the authorization + auth: + # Prometheus Push Gateway + prometheuspush: + # username + username: + # password + password: # Settings related to the binding command bind: # The binding specification of the SUT in the : format diff --git a/packages/caliper-core/lib/common/prometheus/prometheus-push-client.js b/packages/caliper-core/lib/common/prometheus/prometheus-push-client.js deleted file mode 100644 index 62a3248c76..0000000000 --- a/packages/caliper-core/lib/common/prometheus/prometheus-push-client.js +++ /dev/null @@ -1,146 +0,0 @@ -/* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -'use strict'; - -const url = require('url'); -const http = require('http'); -const https = require('https'); - -const Logger = require('../utils/caliper-utils').getLogger('prometheus-push-client'); - -const BASE = 'metrics/job/caliper/'; -const POST = 'POST'; -const DELETE = 'DELETE'; - -/** - * PrometheusClient - client communication with Prometheus metrics through the Push Gateway - */ -class PrometheusPushClient { - - /** - * Constructor for client - * @param {String} gatewayURL the push gateway URL - */ - constructor(gatewayURL) { - this.gatewayURL = gatewayURL; - } - - /** - * Check if gateway has been set - * @returns {Boolean} true if gateway set, otherwise false - */ - gatewaySet(){ - const isSet = this.gatewayURL ? true : false; - return isSet; - } - - /** - * Set the gateway - * @param {String} gatewayURL the push gateway URL - */ - setGateway(gatewayURL) { - this.gatewayURL = gatewayURL; - } - - /** - * Configure the target for the push - * @param {String} testLabel the benchmark test name to store under - * @param {number} testRound the test round to store under - * @param {number} clientId the clientId to store under - */ - configureTarget(testLabel, testRound, clientId) { - const testPath = `instance/${testLabel}/round/${testRound.toString()}/client/${clientId.toString()}`; - const target = url.resolve(this.gatewayURL, BASE + testPath); - this.requestParams = url.parse(target); - this.httpModule = this.isHttps(this.requestParams.href) ? https : http; - Logger.debug(`Prometheus push client configured to target ${this.requestParams.href}`); - } - - /** - * Push a message to the Prometheus gateway - * @param {String} key the key to store the information - * @param {String} value the value to persist - * @param {String[]} tags the tags to use when persisting - */ - push(key, value, tags) { - let body; - if (tags) { - body = `${key}{${tags.join(',')}} ${value}`; - } else { - body = `${key} ${value}`; - } - this.useGateway(POST, body); - } - - /** - * Delete everything under the path within the PushGateway for the current configuration - */ - delete(){ - this.useGateway(DELETE, null); - } - - /** - * Send message on gateway - * @param {String} method the method type [POST | DELETE] - * @param {String} body the body to send - */ - useGateway(method, body) { - Logger.debug(`Prometheus client sending body ${body} to target ${this.requestParams.href}`); - // Convert body to binary, the newline is important - body = Buffer.from(body + '\n', 'binary'); - - // Assign request options - const options = Object.assign(this.requestParams, { - method, - headers: { - 'Content-Type': 'application/octet-stream', - 'Content-Length': body.length - } - }); - - // Make the request - const req = this.httpModule.request(options, res => { - let body = ''; - res.setEncoding('utf8'); - res.on('data', chunk => { - body += chunk; - }); - res.on('end', () => { - if(body) { - Logger.info('PushGateway Response: ' + body); - } - }); - }); - req.on('error', err => { - Logger.error(err); - }); - - // send and end - req.write(body); - req.end(); - } - - /** - * Check if we are using http or https - * @param {*} href the passed Href - * @returns {Boolean} true if https - */ - isHttps(href) { - return href.search(/^https/) !== -1; - } - -} - -module.exports = PrometheusPushClient; diff --git a/packages/caliper-core/lib/common/utils/caliper-utils.js b/packages/caliper-core/lib/common/utils/caliper-utils.js index 6e4d7561c4..74aa73f092 100644 --- a/packages/caliper-core/lib/common/utils/caliper-utils.js +++ b/packages/caliper-core/lib/common/utils/caliper-utils.js @@ -14,13 +14,15 @@ 'use strict'; +const loggingUtil = require('./logging-util.js'); +const Config = require('../config/config-util'); + const {exec, spawn} = require('child_process'); -const path = require('path'); -require('winston-daily-rotate-file'); const fs = require('fs'); const yaml = require('js-yaml'); -const loggingUtil = require('./logging-util.js'); -const Config = require('../config/config-util'); +const path = require('path'); +const url = require('url'); +require('winston-daily-rotate-file'); const BuiltinConnectors = new Map([ ['burrow', '@hyperledger/caliper-burrow'], @@ -588,6 +590,25 @@ class CaliperUtils { return value / 1000; } + /** + * Augment the passed URL with basic auth if the settings are present + * @param {string} urlPath the URL to augment + * @param {string} component the component being augmented + * @returns {string} the URL to be used, which may have been augmented with basic auth + */ + static augmentUrlWithBasicAuth(urlPath, component) { + const username = Config.get(Config.keys.Auth[component].UserName, undefined); + const password = Config.get(Config.keys.Auth[component].Password, undefined); + if (username && password) { + const myURL = new url.URL(urlPath); + myURL.username = username; + myURL.password = password; + return url.format(myURL); + } else { + return urlPath; + } + } + } module.exports = CaliperUtils; diff --git a/packages/caliper-core/lib/common/utils/constants.js b/packages/caliper-core/lib/common/utils/constants.js index a748a822ef..78479d6743 100644 --- a/packages/caliper-core/lib/common/utils/constants.js +++ b/packages/caliper-core/lib/common/utils/constants.js @@ -46,5 +46,8 @@ module.exports = { TxsSubmitted: 'txsSubmitted', TxsFinished: 'txsFinished' } + }, + AuthComponents: { + PushGateway: 'PrometheusPush' } }; diff --git a/packages/caliper-core/lib/worker/tx-observers/prometheus-push-tx-observer.js b/packages/caliper-core/lib/worker/tx-observers/prometheus-push-tx-observer.js index a42e07f743..02ff9d5fd3 100644 --- a/packages/caliper-core/lib/worker/tx-observers/prometheus-push-tx-observer.js +++ b/packages/caliper-core/lib/worker/tx-observers/prometheus-push-tx-observer.js @@ -15,8 +15,13 @@ 'use strict'; const TxObserverInterface = require('./tx-observer-interface'); -const PrometheusClient = require('../../common/prometheus/prometheus-push-client'); const CaliperUtils = require('../../common/utils/caliper-utils'); +const Constants = require('../../common/utils/constants'); + +const prometheus = require('prom-client'); +const promGcStats = require('prometheus-gc-stats'); + +const Logger = CaliperUtils.getLogger('prometheus-push-tx-observer'); /** * Prometheus TX observer used to maintain Prometheus metrics for the push-based scenario (through a push gateway). @@ -33,13 +38,65 @@ class PrometheusPushTxObserver extends TxObserverInterface { this.sendInterval = options && options.sendInterval || 1000; this.intervalObject = undefined; - this.prometheusClient = new PrometheusClient(); - this.prometheusClient.setGateway(options.push_url); + // do not use global registry to avoid conflicts with other potential prom-based observers + this.registry = new prometheus.Registry(); + + // automatically apply default internal and user supplied labels + this.defaultLabels = options.defaultLabels || {}; + this.defaultLabels.workerIndex = this.workerIndex; + this.defaultLabels.roundIndex = this.currentRound; + this.defaultLabels.roundLabel = this.roundLabel; + this.registry.setDefaultLabels(this.defaultLabels); + + // Exposed metrics + this.counterTxSubmitted = new prometheus.Counter({ + name: 'caliper_tx_submitted', + help: 'The total number of submitted transactions.', + registers: [this.registry] + }); + + this.counterTxFinished = new prometheus.Counter({ + name: 'caliper_tx_finished', + help: 'The total number of finished transactions.', + labelNames: ['final_status'], + registers: [this.registry] + }); + + // configure buckets + let buckets = prometheus.linearBuckets(0.1, 0.5, 10); // default + if (options.histogramBuckets) { + if (options.histogramBuckets.explicit) { + buckets = options.histogramBuckets.explicit; + } else if (options.histogramBuckets.linear) { + let linear = options.histogramBuckets.linear; + buckets = prometheus.linearBuckets(linear.start, linear.width, linear.count); + } else if (options.histogramBuckets.exponential) { + let exponential = options.histogramBuckets.exponential; + buckets = prometheus.exponentialBuckets(exponential.start, exponential.factor, exponential.count); + } + } + + this.histogramLatency = new prometheus.Histogram({ + name: 'caliper_tx_e2e_latency', + help: 'The histogram of end-to-end transaction latencies in seconds.', + labelNames: ['final_status'], + buckets, + registers: [this.registry] + }); + + // setting an interval enables the default metric collection + if (this.processMetricCollectInterval) { + this.processMetricHandle = prometheus.collectDefaultMetrics({ + register: this.registry, + timestamps: false, + timeout: this.processMetricCollectInterval + }); + const startGcStats = promGcStats(this.registry); + startGcStats(); + } - this.internalStats = { - previouslyCompletedTotal: 0, - previouslySubmittedTotal: 0 - }; + const url = CaliperUtils.augmentUrlWithBasicAuth(options.pushUrl, Constants.AuthComponents.PushGateway); + this.prometheusClient = new prometheus.Pushgateway(url, null, this.registry); } /** @@ -47,31 +104,11 @@ class PrometheusPushTxObserver extends TxObserverInterface { * @private */ async _sendUpdate() { - const stats = super.getCurrentStatistics(); - this.prometheusClient.configureTarget(stats.getRoundLabel(), stats.getRoundIndex(), stats.getWorkerIndex()); - - // Observer based requirements - this.prometheusClient.push('caliper_txn_success', stats.getTotalSuccessfulTx()); - this.prometheusClient.push('caliper_txn_failure', stats.getTotalFailedTx()); - this.prometheusClient.push('caliper_txn_pending', stats.getTotalSubmittedTx() - stats.getTotalFinishedTx()); - - // TxStats based requirements, existing behaviour batches results bounded within txUpdateTime - const completedTransactions = stats.getTotalSuccessfulTx() + stats.getTotalFailedTx(); - const submittedTransactions = stats.getTotalSubmittedTx(); - - const batchCompletedTransactions = completedTransactions - this.internalStats.previouslyCompletedTotal; - const batchTPS = (batchCompletedTransactions/this.sendInterval)*1000; // txUpdate is in ms - - const batchSubmittedTransactions = submittedTransactions - this.internalStats.previouslyCompletedTotal; - const batchSubmitTPS = (batchSubmittedTransactions/this.sendInterval)*1000; // txUpdate is in ms - const latency = (stats.getTotalLatencyForFailed() + stats.getTotalLatencyForSuccessful()) / completedTransactions; - - this.prometheusClient.push('caliper_tps', batchTPS); - this.prometheusClient.push('caliper_latency', latency/1000); - this.prometheusClient.push('caliper_txn_submit_rate', batchSubmitTPS); - - this.internalStats.previouslyCompletedTotal = batchCompletedTransactions; - this.internalStats.previouslyCompletedTotal = batchSubmittedTransactions; + this.prometheusClient.pushAdd({jobName: 'workers'}, function(err, _resp, _body) { + if (err) { + Logger.error(`Error during push to prometheus: ${err.stack}`); + } + }); } /** @@ -81,6 +118,13 @@ class PrometheusPushTxObserver extends TxObserverInterface { */ async activate(roundIndex, roundLabel) { await super.activate(roundIndex, roundLabel); + + // update worker and round metadata + this.defaultLabels.workerIndex = this.workerIndex; + this.defaultLabels.roundIndex = this.currentRound; + this.defaultLabels.roundLabel = this.roundLabel; + this.registry.setDefaultLabels(this.defaultLabels); + this.intervalObject = setInterval(async () => { await this._sendUpdate(); }, this.sendInterval); } @@ -89,19 +133,40 @@ class PrometheusPushTxObserver extends TxObserverInterface { */ async deactivate() { await super.deactivate(); - - this.internalStats = { - previouslyCompletedTotal: 0, - previouslySubmittedTotal: 0 - }; + this.counterTxSubmitted.reset(); + this.counterTxFinished.reset(); + this.histogramLatency.reset(); + this.registry.resetMetrics(); if (this.intervalObject) { clearInterval(this.intervalObject); + await this._sendUpdate(); + } + } - this.prometheusClient.push('caliper_txn_success', 0); - this.prometheusClient.push('caliper_txn_failure', 0); - this.prometheusClient.push('caliper_txn_pending', 0); - await CaliperUtils.sleep(this.sendInterval); + /** + * Called when TXs are submitted. + * @param {number} count The number of submitted TXs. Can be greater than one for a batch of TXs. + */ + txSubmitted(count) { + this.counterTxSubmitted.inc(count); + } + + /** + * Called when TXs are finished. + * @param {TxStatus | TxStatus[]} results The result information of the finished TXs. Can be a collection of results for a batch of TXs. + */ + txFinished(results) { + if (Array.isArray(results)) { + for (let result of results) { + // pass/fail status from result.GetStatus() + this.counterTxFinished.labels(result.GetStatus()).inc(); + this.histogramLatency.labels(result.GetStatus()).observe(result.GetTimeFinal() - result.GetTimeCreate()); + } + } else { + // pass/fail status from result.GetStatus() + this.counterTxFinished.labels(results.GetStatus()).inc(); + this.histogramLatency.labels(results.GetStatus()).observe((results.GetTimeFinal() - results.GetTimeCreate())/1000); } } diff --git a/packages/caliper-core/test/common/utils/caliper-utils.js b/packages/caliper-core/test/common/utils/caliper-utils.js new file mode 100644 index 0000000000..8baf9a59e4 --- /dev/null +++ b/packages/caliper-core/test/common/utils/caliper-utils.js @@ -0,0 +1,62 @@ +/* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +'use strict'; + +const CaliperUtils = require('../../../lib/common/utils/caliper-utils'); +const ConfigUtil = require('../../../lib/common/config/config-util'); +const Constants = require('../../../lib/common/utils/constants'); + +const chai = require('chai'); +chai.should(); + +describe('caliper utilities', () => { + + + describe('#augmentUrlWithBasicAuth', () => { + const myHttpUrl = 'http://test.com:9090'; + const myHttpsUrl = 'https://test.com:9090'; + + afterEach(() => { + ConfigUtil.set(ConfigUtil.keys.Auth[Constants.AuthComponents.PushGateway].UserName, undefined); + ConfigUtil.set(ConfigUtil.keys.Auth[Constants.AuthComponents.PushGateway].Password, undefined); + }); + + it('should return the unaltered URL if no basic auth parameters for the component are detected', () => { + CaliperUtils.augmentUrlWithBasicAuth(myHttpUrl, Constants.AuthComponents.PushGateway).should.equal(myHttpUrl); + }); + + it('should throw if the url is invalid', () => { + + (() => { + ConfigUtil.set(ConfigUtil.keys.Auth[Constants.AuthComponents.PushGateway].UserName, 'penguin'); + ConfigUtil.set(ConfigUtil.keys.Auth[Constants.AuthComponents.PushGateway].Password, 'madagascar'); + CaliperUtils.augmentUrlWithBasicAuth('badUrl', Constants.AuthComponents.PushGateway); + }).should.throw('Invalid URL: badUrl'); + }); + + it('should augment http with basic auth', () => { + ConfigUtil.set(ConfigUtil.keys.Auth[Constants.AuthComponents.PushGateway].UserName, 'penguin'); + ConfigUtil.set(ConfigUtil.keys.Auth[Constants.AuthComponents.PushGateway].Password, 'madagascar'); + CaliperUtils.augmentUrlWithBasicAuth(myHttpUrl, Constants.AuthComponents.PushGateway).should.equal('http://penguin:madagascar@test.com:9090/'); + }); + + it('should augment https with basic auth', () => { + ConfigUtil.set(ConfigUtil.keys.Auth[Constants.AuthComponents.PushGateway].UserName, 'penguin'); + ConfigUtil.set(ConfigUtil.keys.Auth[Constants.AuthComponents.PushGateway].Password, 'madagascar'); + CaliperUtils.augmentUrlWithBasicAuth(myHttpsUrl, Constants.AuthComponents.PushGateway).should.equal('https://penguin:madagascar@test.com:9090/'); + }); + }); + +});