From 479d789b31ab823cb5422606c302030a386658ca Mon Sep 17 00:00:00 2001 From: "nkl199@yahoo.co.uk" Date: Tue, 28 Jul 2020 13:29:59 +0100 Subject: [PATCH] Add txObservers and make use of TXN Stats Collector Signed-off-by: nkl199@yahoo.co.uk --- .../lib/launch/lib/launchWorker.js | 4 +- packages/caliper-core/index.js | 2 +- .../caliper-core/lib/common/config/Config.js | 12 +- .../lib/common/config/default.yaml | 17 +- .../core/transaction-statistics-collector.js | 424 ++++++++++++++ .../lib/common/core/transaction-statistics.js | 200 ------- .../lib/common/messages/testMessage.js | 2 +- .../lib/common/messages/txUpdateMessage.js | 1 - .../lib/common/utils/caliper-utils.js | 9 + .../lib/common/utils/circular-array.js | 46 -- .../lib/manager/caliper-engine.js | 2 +- .../lib/manager/monitors/monitor-docker.js | 15 +- .../lib/manager/monitors/monitor-interface.js | 11 +- .../lib/manager/monitors/monitor-process.js | 74 ++- .../manager/monitors/monitor-prometheus.js | 43 +- .../orchestrators/monitor-orchestrator.js | 34 +- .../orchestrators/round-orchestrator.js | 37 +- .../orchestrators/worker-orchestrator.js | 41 +- .../caliper-core/lib/manager/report/report.js | 148 +---- ...{local-observer.js => default-observer.js} | 147 ++--- .../manager/test-observers/null-observer.js | 18 +- .../test-observers/observer-interface.js | 12 +- .../test-observers/prometheus-observer.js | 126 ----- .../manager/test-observers/test-observer.js | 21 +- .../caliper-core/lib/worker/caliper-worker.js | 365 +++--------- .../lib/worker/rate-control/compositeRate.js | 269 ++++----- .../lib/worker/rate-control/fixedBacklog.js | 136 ----- .../worker/rate-control/fixedFeedbackRate.js | 126 ++--- .../lib/worker/rate-control/fixedLoad.js | 112 ++++ .../lib/worker/rate-control/fixedRate.js | 87 +-- .../lib/worker/rate-control/linearRate.js | 111 ++-- .../lib/worker/rate-control/maxRate.js | 243 ++++---- .../lib/worker/rate-control/noRate.js | 73 +-- .../lib/worker/rate-control/rateControl.js | 82 ++- .../lib/worker/rate-control/rateInterface.js | 55 +- .../lib/worker/rate-control/recordRate.js | 128 ++--- .../lib/worker/rate-control/replayRate.js | 147 ++--- .../tx-observers/internal-tx-observer.js | 81 +++ .../tx-observers/logging-tx-observer.js | 79 +++ .../prometheus-push-tx-observer.js | 121 ++++ .../tx-observers/tx-observer-dispatch.js | 134 +++++ .../tx-observers/tx-observer-interface.js | 104 ++++ ...e-handler.js => worker-message-handler.js} | 20 +- .../manager/monitors/monitor-prometheus.js | 20 - .../orchestrators/worker-orchestrator.js | 41 +- .../test/manager/report/report.js | 101 ++-- .../test/worker/rate-control/fixedBacklog.js | 225 -------- .../worker/rate-control/fixedFeedbackRate.js | 253 +++------ .../test/worker/rate-control/fixedLoad.js | 183 ++++++ .../test/worker/rate-control/fixedRate.js | 108 ++-- .../test/worker/rate-control/linearRate.js | 225 +++++--- .../test/worker/rate-control/maxRate.js | 526 ++++++++---------- .../test/worker/rate-control/noRate.js | 83 ++- .../besu_tests/phase1/benchconfig.yaml | 3 - .../besu_tests/phase2/benchconfig.yaml | 3 - .../besu_tests/phase3/benchconfig.yaml | 3 - .../ethereum_tests/benchconfig.yaml | 4 +- .../benchconfig.yaml | 5 - .../benchconfig.yaml | 5 - .../fabric_tests/phase2/benchconfig.yaml | 14 +- .../fabric_tests/phase3/benchconfig.yaml | 18 +- .../fabric_tests/phase4/benchconfig.yaml | 38 +- .../fabric_tests/phase5/benchconfig.yaml | 53 +- .../fabric_tests/phase6/benchconfig.yaml | 22 +- .../fabric_tests/phase7/benchconfig.yaml | 8 - .../fisco-bcos_tests/benchconfig.yaml | 3 - .../generator_tests/fabric/caliper.yaml | 2 +- .../generator_tests/fabric/run.sh | 2 +- .../iroha_tests/benchconfig.yaml | 3 - .../sawtooth_tests/benchconfig.yaml | 4 - .../generators/benchmark/index.js | 15 +- .../benchmark/templates/config.yaml | 5 +- .../test/benchmark/config.js | 7 +- 73 files changed, 2900 insertions(+), 3001 deletions(-) create mode 100644 packages/caliper-core/lib/common/core/transaction-statistics-collector.js delete mode 100644 packages/caliper-core/lib/common/core/transaction-statistics.js delete mode 100644 packages/caliper-core/lib/common/utils/circular-array.js rename packages/caliper-core/lib/manager/test-observers/{local-observer.js => default-observer.js} (57%) delete mode 100644 packages/caliper-core/lib/manager/test-observers/prometheus-observer.js delete mode 100644 packages/caliper-core/lib/worker/rate-control/fixedBacklog.js create mode 100644 packages/caliper-core/lib/worker/rate-control/fixedLoad.js create mode 100644 packages/caliper-core/lib/worker/tx-observers/internal-tx-observer.js create mode 100644 packages/caliper-core/lib/worker/tx-observers/logging-tx-observer.js create mode 100644 packages/caliper-core/lib/worker/tx-observers/prometheus-push-tx-observer.js create mode 100644 packages/caliper-core/lib/worker/tx-observers/tx-observer-dispatch.js create mode 100644 packages/caliper-core/lib/worker/tx-observers/tx-observer-interface.js rename packages/caliper-core/lib/worker/{message-handler.js => worker-message-handler.js} (93%) delete mode 100644 packages/caliper-core/test/worker/rate-control/fixedBacklog.js create mode 100644 packages/caliper-core/test/worker/rate-control/fixedLoad.js diff --git a/packages/caliper-cli/lib/launch/lib/launchWorker.js b/packages/caliper-cli/lib/launch/lib/launchWorker.js index 87d3323cc8..f45ff449ce 100644 --- a/packages/caliper-cli/lib/launch/lib/launchWorker.js +++ b/packages/caliper-cli/lib/launch/lib/launchWorker.js @@ -14,7 +14,7 @@ 'use strict'; -const { CaliperUtils, ConfigUtil, Constants, MessageHandler } = require('@hyperledger/caliper-core'); +const { CaliperUtils, ConfigUtil, Constants, WorkerMessageHandler } = require('@hyperledger/caliper-core'); const BindCommon = require('./../../lib/bindCommon'); const logger = CaliperUtils.getLogger('cli-launch-worker'); @@ -79,7 +79,7 @@ class LaunchWorker { * @type {MessengerInterface} */ const messenger = messengerFactory({}); - const messageHandler = new MessageHandler(messenger, connectorFactory); + const messageHandler = new WorkerMessageHandler(messenger, connectorFactory); await messenger.initialize(); await messenger.configureProcessInstances([process]); diff --git a/packages/caliper-core/index.js b/packages/caliper-core/index.js index f98a338606..0a11e811a5 100644 --- a/packages/caliper-core/index.js +++ b/packages/caliper-core/index.js @@ -21,7 +21,7 @@ module.exports.TxStatus = require('./lib/common/core/transaction-status'); module.exports.CaliperUtils = require('./lib/common/utils/caliper-utils'); module.exports.Version = require('./lib/common/utils/version'); module.exports.ConfigUtil = require('./lib/common/config/config-util'); -module.exports.MessageHandler = require('./lib/worker/message-handler'); +module.exports.WorkerMessageHandler = require('./lib/worker/worker-message-handler'); module.exports.MessengerInterface = require('./lib/common/messengers/messenger-interface'); module.exports.CaliperEngine = require('./lib/manager/caliper-engine'); module.exports.MonitorOrchestrator = require('./lib/manager/orchestrators/monitor-orchestrator'); diff --git a/packages/caliper-core/lib/common/config/Config.js b/packages/caliper-core/lib/common/config/Config.js index 38fc3e47ae..98f2616d53 100644 --- a/packages/caliper-core/lib/common/config/Config.js +++ b/packages/caliper-core/lib/common/config/Config.js @@ -37,13 +37,19 @@ const keys = { Transparency: 'caliper-report-charting-transparency' } }, + Progress: { + Reporting: { + Enabled: 'caliper-progress-reporting-enabled', + Interval: 'caliper-progress-reporting-interval' + } + }, Workspace: 'caliper-workspace', ProjectConfig: 'caliper-projectconfig', UserConfig: 'caliper-userconfig', MachineConfig: 'caliper-machineconfig', BenchConfig: 'caliper-benchconfig', NetworkConfig: 'caliper-networkconfig', - TxUpdateTime: 'caliper-txupdatetime', + MonitorConfig: 'caliper-monitorconfig', LoggingRoot: 'caliper-logging', Logging: { Template: 'caliper-logging-template', @@ -92,7 +98,9 @@ const keys = { Method: 'caliper-worker-communication-method', Address: 'caliper-worker-communication-address', }, - MaxTxPromises: 'caliper-worker-maxtxpromises' + Update: { + Interval: 'caliper-worker-update-interval' + } }, Flow: { Skip: { diff --git a/packages/caliper-core/lib/common/config/default.yaml b/packages/caliper-core/lib/common/config/default.yaml index 5373ca001b..73229dd354 100644 --- a/packages/caliper-core/lib/common/config/default.yaml +++ b/packages/caliper-core/lib/common/config/default.yaml @@ -50,8 +50,14 @@ caliper: benchconfig: # Path to the blockchain configuration file that contains information required to interact with the SUT networkconfig: - # Sets the frequency of the progress reports in milliseconds - txupdatetime: 5000 + # Configurations related to caliper test progress + progress: + # Progress reports + reporting: + # Enable the reporting + enabled: true + # Report frequency + interval: 5000 # Configurations related to the logging mechanism logging: # Specifies the message structure through placeholders @@ -114,15 +120,16 @@ caliper: worker: # Indicate if workers are in distributed mode remote: false - # Polling interval to use once created, in milliseconds - pollinterval: 5000 # Worker communication details communication: # Method used (process | mqtt) method: process # Address used for mqtt communications address: mqtt://localhost:1883 - maxtxpromises: 100 + # Worker update configuration + update: + # update interval for sending round statistics to the manager + interval: 1000 # Caliper flow options flow: # Skip options diff --git a/packages/caliper-core/lib/common/core/transaction-statistics-collector.js b/packages/caliper-core/lib/common/core/transaction-statistics-collector.js new file mode 100644 index 0000000000..b2fc38110a --- /dev/null +++ b/packages/caliper-core/lib/common/core/transaction-statistics-collector.js @@ -0,0 +1,424 @@ +/* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +'use strict'; + +/** + * Encapsulates TX statistics for a given worker node for a given round. + * + * @property {TransactionStatisticsCollector[]} subCollectors Collection of TX statistics collectors processing only a subset of TXs. + * @property {boolean} active Indicates whether the collector is active, i.e. it processes TX events. + * @property {{metadata: {workerIndex: number, roundIndex: number, roundStartTime: number, roundFinishTime: number}, txCounters: {totalSubmitted: number, totalFinished: number, totalSuccessful: number, totalFailed: number}, timestamps: {firstCreateTime: number, lastCreateTime: number, firstFinishTime: number, lastFinishTime: number}, latency: {successful: {min: number, max: number, total: number}, failed: {min: number, max: number, total: number}}}} stats The different cumulative TX statistics. + */ +class TransactionStatisticsCollector { + /** + * Initializes the instance. + * @param {number} workerIndex The index of the worker process. + * @param {number} roundIndex The index of the round. + * @param {string} roundLabel The roundLabel name. + */ + constructor(workerIndex, roundIndex, roundLabel) { + this.subCollectors = []; + this.active = false; + + this.stats = { + metadata: { + workerIndex, + roundIndex, + roundLabel, + roundStartTime: 0, // deactivated + roundFinishTime: 0 + }, + txCounters: { + totalSubmitted: 0, + totalFinished: 0, + totalSuccessful: 0, + totalFailed: 0 + }, + timestamps: { + firstCreateTime: Number.MAX_SAFE_INTEGER, // best default value for min + lastCreateTime: 0, + firstFinishTime: Number.MAX_SAFE_INTEGER, // best default value for min + lastFinishTime: 0 + }, + latency: { + successful: { + min: Number.MAX_SAFE_INTEGER, // best default value for min + max: 0, + total: 0 + }, + failed: { + min: Number.MAX_SAFE_INTEGER, // best default value for min + max: 0, + total: 0 + } + } + }; + } + + /** + * Updates the TX statistics with the new result. + * @param {TxStatus} txResult The TX status/information. + * @private + */ + _updateStatistics(txResult) { + this.stats.txCounters.totalFinished +=1; + + // updating create time stats + let createTime = txResult.GetTimeCreate(); + this.stats.timestamps.firstCreateTime = + Math.min(createTime, this.stats.timestamps.firstCreateTime); + this.stats.timestamps.lastCreateTime = + Math.max(createTime, this.stats.timestamps.lastCreateTime); + + // updating finish time stats + let finishTime = txResult.GetTimeFinal(); + this.stats.timestamps.firstFinishTime = + Math.min(finishTime, this.stats.timestamps.firstFinishTime); + this.stats.timestamps.lastFinishTime = + Math.max(finishTime, this.stats.timestamps.lastFinishTime); + + let latency = finishTime - createTime; + + // separate stats for successful and failed TXs + if (txResult.IsCommitted()) { + this.stats.txCounters.totalSuccessful += 1; + + // latency stats + this.stats.latency.successful.min = + Math.min(latency, this.stats.latency.successful.min); + this.stats.latency.successful.max = + Math.max(latency, this.stats.latency.successful.max); + this.stats.latency.successful.total += latency; + + } else { + this.stats.txCounters.totalFailed += 1; + + // latency stats + this.stats.latency.failed.min = + Math.min(latency, this.stats.latency.failed.min); + this.stats.latency.failed.max = + Math.max(latency, this.stats.latency.failed.max); + this.stats.latency.failed.total += latency; + } + } + + //////////// + // Getters + //////////// + + /** + * Get the 0-based worker node index. + * @return {number} The worker node index. + */ + getWorkerIndex() { + return this.stats.metadata.workerIndex; + } + + /** + * Get the 0-based round index. + * @return {number} The round index. + */ + getRoundIndex() { + return this.stats.metadata.roundIndex; + } + + /** + * Return the string round label name + * @return {string} the round label name + */ + getRoundLabel() { + return this.stats.metadata.roundLabel; + } + + /** + * Get the start time of the round. + * @return {number} The epoch in milliseconds when the round was started. + */ + getRoundStartTime() { + return this.stats.metadata.roundStartTime; + } + + /** + * Get the finish time of the round. + * @return {number} The epoch in milliseconds when the round was finished. + */ + getRoundFinishTime() { + return this.stats.metadata.roundFinishTime; + } + + /** + * Get the total number of submitted TXs. + * @return {number} The total number of submitted TXs. + */ + getTotalSubmittedTx() { + return this.stats.txCounters.totalSubmitted; + } + + /** + * Get the total number of finished TXs. + * @return {number} The total number of finished TXs. + */ + getTotalFinishedTx() { + return this.stats.txCounters.totalFinished; + } + + /** + * Get the total number of successful TXs. + * @return {number} The total number of successful TXs. + */ + getTotalSuccessfulTx() { + return this.stats.txCounters.totalSuccessful; + } + + /** + * Get the total number of failed TXs. + * @return {number} The total number of failed TXs. + */ + getTotalFailedTx() { + return this.stats.txCounters.totalFailed; + } + + /** + * Get the create time of the first submitted TX. + * @return {number} The epoch in milliseconds when the first TX was submitted. + */ + getFirstCreateTime() { + return this.stats.timestamps.firstCreateTime; + } + + /** + * Get the create time of the last submitted TX. + * @return {number} The epoch in milliseconds when the last TX was submitted. + */ + getLastCreateTime() { + return this.stats.timestamps.lastCreateTime; + } + + /** + * Get the finish time of the first finished TX. + * @return {number} The epoch in milliseconds when the first TX was finished. + */ + getFirstFinishTime() { + return this.stats.timestamps.firstFinishTime; + } + + /** + * Get the finish time of the last finished TX. + * @return {number} The epoch in milliseconds when the last TX was finished. + */ + getLastFinishTime() { + return this.stats.timestamps.lastFinishTime; + } + + /** + * Get the shortest latency for successful TXs. + * @return {number} The shortest latency for successful TXs. + */ + getMinLatencyForSuccessful() { + return this.stats.latency.successful.min; + } + + /** + * Get the longest latency for successful TXs. + * @return {number} The longest latency for successful TXs. + */ + getMaxLatencyForSuccessful() { + return this.stats.latency.successful.max; + } + + /** + * Get the total/summed latency for successful TXs. + * @return {number} The total/summed latency for successful TXs. + */ + getTotalLatencyForSuccessful() { + return this.stats.latency.successful.total; + } + + /** + * Get the shortest latency for failed TXs. + * @return {number} The shortest latency for failed TXs. + */ + getMinLatencyForFailed() { + return this.stats.latency.failed.min; + } + + /** + * Get the longest latency for failed TXs. + * @return {number} The longest latency for failed TXs. + */ + getMaxLatencyForFailed() { + return this.stats.latency.failed.max; + } + + /** + * Get the total/summed latency for failed TXs. + * @return {number} The total/summed latency for failed TXs. + */ + getTotalLatencyForFailed() { + return this.stats.latency.failed.total; + } + + /** + * Get a copy of the cumulative TX statistics. + * @return {{metadata: {workerIndex: number, roundIndex: number, roundStartTime: number, roundFinishTime: number}, txCounters: {totalSubmitted: number, totalFinished: number, totalSuccessful: number, totalFailed: number}, timestamps: {firstCreateTime: number, lastCreateTime: number, firstFinishTime: number, lastFinishTime: number}, latency: {successful: {min: number, max: number, total: number}, failed: {min: number, max: number, total: number}}}} The aggregated TX statistics at the time of the function call. + */ + getCumulativeTxStatistics() { + return JSON.parse(JSON.stringify(this.stats)); + } + + /** + * Create a TX stat collector from the given object. + * @param {{metadata: {workerIndex: number, roundIndex: number, roundStartTime: number, roundFinishTime: number}, txCounters: {totalSubmitted: number, totalFinished: number, totalSuccessful: number, totalFailed: number}, timestamps: {firstCreateTime: number, lastCreateTime: number, firstFinishTime: number, lastFinishTime: number}, latency: {successful: {min: number, max: number, total: number}, failed: {min: number, max: number, total: number}}}} obj The source object. + * @return {TransactionStatisticsCollector} The TX stat collector instance. + */ + static loadFromObject(obj) { + let collector = new TransactionStatisticsCollector(obj.metadata.workerIndex, obj.metadata.roundIndex); + collector.stats = JSON.parse(JSON.stringify(obj)); + return collector; + } + + /** + * Merge the given collector statistics into a single collector statistic. + * @param {TransactionStatisticsCollector[]} collectors The collectors whose current results should be merged. + * @return {TransactionStatisticsCollector} The collector containing the merged results. + */ + static mergeCollectorResults(collectors) { + // snapshot of current results + let currentStats = collectors.map(c => c.getCumulativeTxStatistics()); + + // NOTE: we have a 2D grid of stats, the axis are: "worker" and "round" + // 1) If the stats are all from the same round, and from different workers, then we get a round-level summary + // 2) If the stats are all from the same worker, and from different rounds, then we get a worker-level summary + // 3) If the stats are from all rounds and from all workers, then we get a benchmark-level summary + + let uniqueWorkerIndices = Array.from(new Set(currentStats.map(s => s.metadata.workerIndex))); + let uniqueRoundIndices = Array.from(new Set(currentStats.map(s => s.metadata.roundIndex))); + + // if everything is from the same worker, we can "merge" its index + let workerIndex = uniqueWorkerIndices.length === 1 ? uniqueWorkerIndices[0] : -1; + + // if everything is from the same round, we can "merge" its index + let roundIndex = uniqueRoundIndices.length === 1 ? uniqueRoundIndices[0] : -1; + + const sum = (prev, curr) => prev + curr; + + let mergedStats = { + metadata: { + workerIndex: workerIndex, + roundIndex: roundIndex, + roundStartTime: Math.min(...currentStats.map(s => s.metadata.roundStartTime)), // Remark 1 + roundFinishTime: Math.max(...currentStats.map(s => s.metadata.roundFinishTime)), // Remark 2 + }, + txCounters: { + totalSubmitted: currentStats.map(s => s.txCounters.totalSubmitted).reduce(sum, 0), + totalFinished: currentStats.map(s => s.txCounters.totalFinished).reduce(sum, 0), + totalSuccessful: currentStats.map(s => s.txCounters.totalSuccessful).reduce(sum, 0), + totalFailed: currentStats.map(s => s.txCounters.totalFailed).reduce(sum, 0), + }, + timestamps: { + firstCreateTime: Math.min(...currentStats.map(s => s.timestamps.firstFinishTime)), + lastCreateTime: Math.max(...currentStats.map(s => s.timestamps.lastCreateTime)), + firstFinishTime: Math.min(...currentStats.map(s => s.timestamps.firstFinishTime)), + lastFinishTime: Math.max(...currentStats.map(s => s.timestamps.lastFinishTime)), + }, + latency: { + successful: { + min: Math.min(...currentStats.map(s => s.latency.successful.min)), + max: Math.max(...currentStats.map(s => s.latency.successful.max)), + total: currentStats.map(s => s.latency.successful.total).reduce(sum, 0), + }, + failed: { + min: Math.min(...currentStats.map(s => s.latency.failed.min)), + max: Math.max(...currentStats.map(s => s.latency.failed.max)), + total: currentStats.map(s => s.latency.failed.total).reduce(sum, 0), + } + } + }; + + return TransactionStatisticsCollector.loadFromObject(mergedStats); + } + + /** + * Add a sub-collector to monitor the subset of TX events. + * @param {TransactionStatisticsCollector} collector The sub-collector instance. + */ + addSubCollector(collector) { + this.subCollectors.push(collector); + } + + ////////////// + // TX events + ////////////// + + /** + * Activates the collector and marks the round starting time. + * NOTE: the sub-collectors are not activated. + */ + activate() { + this.stats.metadata.roundStartTime = Date.now(); + this.active = true; + } + + /** + * Called when TXs are submitted. Updates the related statistics. + * @param {number} count The number of submitted TXs. Can be greater than one for a batch of TXs. + */ + txSubmitted(count) { + if (!this.active) { + return; + } + + this.stats.txCounters.totalSubmitted += count; + for (let subcollector of this.subCollectors) { + subcollector.txSubmitted(count); + } + } + + /** + * Called when TXs are finished. Updates the related statistics. + * @param {TxStatus | TxStatus[]} results The result information of the finished TXs. Can be a collection of results for a batch of TXs. + */ + txFinished(results) { + if (!this.active) { + return; + } + + if (Array.isArray(results)) { + let relevantResults = results.filter(r => r.GetTimeCreate() > this.getRoundStartTime()); + for (let result of relevantResults) { + this._updateStatistics(result); + } + for (let subcollector of this.subCollectors) { + subcollector.txFinished(relevantResults); + } + } else if (results.GetTimeCreate() > this.getRoundStartTime()) { + this._updateStatistics(results); + for (let subcollector of this.subCollectors) { + subcollector.txFinished(results); + } + } + } + + /** + * Deactivates the collector and marks the round finish time. + * NOTE: the sub-collectors are not deactivated. + */ + deactivate() { + this.stats.metadata.roundFinishTime = Date.now(); + this.active = false; + } +} + +module.exports = TransactionStatisticsCollector; diff --git a/packages/caliper-core/lib/common/core/transaction-statistics.js b/packages/caliper-core/lib/common/core/transaction-statistics.js deleted file mode 100644 index f24af419dd..0000000000 --- a/packages/caliper-core/lib/common/core/transaction-statistics.js +++ /dev/null @@ -1,200 +0,0 @@ -/* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -'use strict'; - -const Logger = require('../utils/caliper-utils').getLogger('transaction-statistics'); - -/** - * Class for working on Transaction statistics - */ -class TransactionStatistics { - - /** - * create a 'null' txStatistics object - * @return {JSON} 'null' txStatistics object - */ - static createNullDefaultTxStats() { - return { - succ: 0, - fail: 0 - }; - } - - /** - * Calculate the default transaction statistics - * @param {Array} resultArray array of txStatus - * @param {Boolean} detail indicates whether to keep detailed information - * @return {JSON} txStatistics JSON object - */ - static getDefaultTxStats(resultArray, detail) { - let succ = 0, fail = 0, delay = 0; - let minFinal, maxFinal, minCreate, maxCreate; - let maxLastFinal; - let minDelay = 100000, maxDelay = 0; - let delays = []; - let sTPTotal = 0; - let sTTotal = 0; - let invokeTotal = 0; - for (let i = 0; i < resultArray.length; i++) { - let stat = resultArray[i]; - sTPTotal = sTPTotal + stat.Get('sTP'); - sTTotal = sTTotal + stat.Get('sT'); - invokeTotal += stat.Get('invokeLatency'); - let create = stat.GetTimeCreate(); - - if (typeof minCreate === 'undefined') { - minCreate = create; - maxCreate = create; - } else { - if (create < minCreate) { - minCreate = create; - } - if (create > maxCreate) { - maxCreate = create; - } - } - - if (stat.IsCommitted()) { - succ++; - let final = stat.GetTimeFinal(); - let d = (final - create) / 1000; - if (typeof minFinal === 'undefined') { - minFinal = final; - maxFinal = final; - } else { - if (final < minFinal) { - minFinal = final; - } - if (final > maxFinal) { - maxFinal = final; - } - } - - delay += d; - if (d < minDelay) { - minDelay = d; - } - if (d > maxDelay) { - maxDelay = d; - } - if (detail) { - delays.push(d); - } - } else { - fail++; - } - - let curFinal = stat.GetTimeFinal(); - if (typeof maxLastFinal === 'undefined') { - maxLastFinal = curFinal; - } else { - if (curFinal > maxLastFinal) { - maxLastFinal = curFinal; - } - } - } - - let stats = { - 'succ': succ, - 'fail': fail, - 'create': { 'min': minCreate / 1000, 'max': maxCreate / 1000 }, // convert to second - 'final': { 'min': minFinal / 1000, 'max': maxFinal / 1000, 'last': maxLastFinal / 1000 }, - 'delay': { 'min': minDelay, 'max': maxDelay, 'sum': delay, 'detail': (detail ? delays : []) }, - 'out': [], - 'sTPTotal': sTPTotal, - 'sTTotal': sTTotal, - 'invokeTotal': invokeTotal, - 'length': resultArray.length - }; - return stats; - } - - /** - * merge an array of default 'txStatistics', the result is in first object of the array - * Note even failed the first object of the array may still be changed - * @param {Array} resultArray txStatistics array - * @return {Number} 0 if failed; otherwise 1 - */ - static mergeDefaultTxStats(resultArray) { - try { - // skip invalid result - let skip = 0; - for (let i = 0; i < resultArray.length; i++) { - let result = resultArray[i]; - - if (!result.hasOwnProperty('succ') || !result.hasOwnProperty('fail') || (result.succ + result.fail) === 0) { - skip++; - } else { - break; - } - } - - if (skip > 0) { - resultArray.splice(0, skip); - } - - if (resultArray.length === 0) { - return 0; - } - - let r = resultArray[0]; - for (let i = 1; i < resultArray.length; i++) { - let v = resultArray[i]; - if (!v.hasOwnProperty('succ') || !v.hasOwnProperty('fail') || (v.succ + v.fail) === 0) { - continue; - } - r.succ += v.succ; - r.fail += v.fail; - r.sTPTotal += v.sTPTotal; - r.sTTotal += v.sTTotal; - r.invokeTotal += v.invokeTotal; - r.length += v.length; - r.out.push.apply(r.out, v.out); - if (v.create.min < r.create.min) { - r.create.min = v.create.min; - } - if (v.create.max > r.create.max) { - r.create.max = v.create.max; - } - if (v.final.min < r.final.min) { - r.final.min = v.final.min; - } - if (v.final.max > r.final.max) { - r.final.max = v.final.max; - } - if (v.final.last > r.final.last) { - r.final.last = v.final.last; - } - if (v.delay.min < r.delay.min) { - r.delay.min = v.delay.min; - } - if (v.delay.max > r.delay.max) { - r.delay.max = v.delay.max; - } - r.delay.sum += v.delay.sum; - for (let j = 0; j < v.delay.detail.length; j++) { - r.delay.detail.push(v.delay.detail[j]); - } - } - return 1; - } - catch (err) { - Logger.error(err); - return 0; - } - } -} - -module.exports = TransactionStatistics; diff --git a/packages/caliper-core/lib/common/messages/testMessage.js b/packages/caliper-core/lib/common/messages/testMessage.js index 2dd81f657d..4f9a846f0d 100644 --- a/packages/caliper-core/lib/common/messages/testMessage.js +++ b/packages/caliper-core/lib/common/messages/testMessage.js @@ -54,7 +54,7 @@ class TestMessage extends Message { * Gets the label of the round. * @return {string} The label of the round. */ - getRoundLabel () { + getRoundLabel() { return this.content.label; } diff --git a/packages/caliper-core/lib/common/messages/txUpdateMessage.js b/packages/caliper-core/lib/common/messages/txUpdateMessage.js index 85079c8ce3..be6ed2d154 100644 --- a/packages/caliper-core/lib/common/messages/txUpdateMessage.js +++ b/packages/caliper-core/lib/common/messages/txUpdateMessage.js @@ -31,7 +31,6 @@ class TxUpdateMessage extends Message { */ constructor(sender, recipients, content, date = undefined, error = undefined) { super(sender, recipients, MessageTypes.TxUpdate, content, date, error); - // the local observers needs this content, deep-deep down this.content.type = MessageTypes.TxUpdate; } diff --git a/packages/caliper-core/lib/common/utils/caliper-utils.js b/packages/caliper-core/lib/common/utils/caliper-utils.js index fc16f59cdf..6e4d7561c4 100644 --- a/packages/caliper-core/lib/common/utils/caliper-utils.js +++ b/packages/caliper-core/lib/common/utils/caliper-utils.js @@ -579,6 +579,15 @@ class CaliperUtils { } + /** + * Convert milliseconds to seconds + * @param {number} value to convert + * @returns {number} the converted value + */ + static millisToSeconds(value) { + return value / 1000; + } + } module.exports = CaliperUtils; diff --git a/packages/caliper-core/lib/common/utils/circular-array.js b/packages/caliper-core/lib/common/utils/circular-array.js deleted file mode 100644 index 504ef24d57..0000000000 --- a/packages/caliper-core/lib/common/utils/circular-array.js +++ /dev/null @@ -1,46 +0,0 @@ -/* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -'use strict'; - -/** - * Create an Array that has a maximum length and will overwrite existing entries when additional items are added to the array - */ -class CircularArray extends Array { - - /** - * Constructor - * @param {number} maxLength maximum length of array - */ - constructor(maxLength) { - super(); - this.pointer = 0; - this.maxLength = maxLength; - } - - /** - * Add entry into array - * @param {any} element the element to add to the array - */ - add(element) { - if (this.length === this.maxLength) { - this[this.pointer] = element; - } else { - this.push(element); - } - this.pointer = (this.pointer + 1) % this.maxLength; - } -} - -module.exports = CircularArray; diff --git a/packages/caliper-core/lib/manager/caliper-engine.js b/packages/caliper-core/lib/manager/caliper-engine.js index bf33077980..4b2006704e 100644 --- a/packages/caliper-core/lib/manager/caliper-engine.js +++ b/packages/caliper-core/lib/manager/caliper-engine.js @@ -152,7 +152,7 @@ class CaliperEngine { // this means that we haven't handled/logged this failure yet if (this.returnCode < 0) { // log full stack - let msg = `Error while performing "test" step: ${err}`; + let msg = `Error while performing "test" step: ${err.stack}`; logger.error(msg); this.returnCode = 6; } diff --git a/packages/caliper-core/lib/manager/monitors/monitor-docker.js b/packages/caliper-core/lib/manager/monitors/monitor-docker.js index 26fdea2b79..4a710c963d 100644 --- a/packages/caliper-core/lib/manager/monitors/monitor-docker.js +++ b/packages/caliper-core/lib/manager/monitors/monitor-docker.js @@ -31,11 +31,10 @@ const SystemInformation = require('systeminformation'); class MonitorDocker extends MonitorInterface { /** * Constructor - * @param {JSON} monitorConfig Configuration object for the monitor - * @param {*} interval resource fetching interval + * @param {JSON} resourceMonitorOptions Configuration options for the monitor */ - constructor(monitorConfig, interval) { - super(monitorConfig, interval); + constructor(resourceMonitorOptions) { + super(resourceMonitorOptions); this.containers = null; this.isReading = false; this.intervalObj = null; @@ -66,9 +65,9 @@ class MonitorDocker extends MonitorInterface { this.containers = []; let filterName = { local: [], remote: {} }; // Split docker items that are local or remote - if (this.monitorConfig.hasOwnProperty('containers')) { - for (let key in this.monitorConfig.containers) { - let container = this.monitorConfig.containers[key]; + if (this.options.hasOwnProperty('containers')) { + for (let key in this.options.containers) { + let container = this.options.containers[key]; if (container.indexOf('http://') === 0) { // Is remote let remote = URL.parse(container, true); @@ -378,7 +377,7 @@ class MonitorDocker extends MonitorInterface { } // Retrieve Chart data - const chartTypes = this.monitorConfig.charting; + const chartTypes = this.options.charting; let chartStats = []; if (chartTypes) { chartStats = ChartBuilder.retrieveChartStats(this.constructor.name, chartTypes, testLabel, resourceStats); diff --git a/packages/caliper-core/lib/manager/monitors/monitor-interface.js b/packages/caliper-core/lib/manager/monitors/monitor-interface.js index bf6e0cd646..07330f69f5 100644 --- a/packages/caliper-core/lib/manager/monitors/monitor-interface.js +++ b/packages/caliper-core/lib/manager/monitors/monitor-interface.js @@ -15,6 +15,8 @@ 'use strict'; +const ConfigUtil = require('../../common/config/config-util.js'); + // TODO: now we record the performance information in local variable, it's better to use db later /** * Interface of resource consumption monitor @@ -22,12 +24,11 @@ class MonitorInterface{ /** * Constructor - * @param {JSON} monitorConfig Configuration object for the monitor - * @param {number} interval Watching interval, in seconds + * @param {JSON} resourceMonitorOptions Configuration options for the monitor */ - constructor(monitorConfig, interval) { - this.monitorConfig = monitorConfig; - this.interval = interval*1000; // convert to ms + constructor(resourceMonitorOptions) { + this.options = resourceMonitorOptions; + this.interval = resourceMonitorOptions.interval ? resourceMonitorOptions.interval*1000 : ConfigUtil.get(ConfigUtil.keys.Progress.Reporting.Interval); } /** diff --git a/packages/caliper-core/lib/manager/monitors/monitor-process.js b/packages/caliper-core/lib/manager/monitors/monitor-process.js index f6f609d80b..28573f674c 100644 --- a/packages/caliper-core/lib/manager/monitors/monitor-process.js +++ b/packages/caliper-core/lib/manager/monitors/monitor-process.js @@ -17,8 +17,8 @@ const MonitorInterface = require('./monitor-interface'); const MonitorUtilities = require('./monitor-utilities'); const Util = require('../../common/utils/caliper-utils'); -const Logger = Util.getLogger('monitor-process'); const ChartBuilder = require('../charts/chart-builder'); +const Logger = Util.getLogger('monitor-process'); const ps = require('ps-node'); const usage = require('pidusage'); @@ -29,33 +29,13 @@ const usage = require('pidusage'); class MonitorProcess extends MonitorInterface { /** * Constructor - * @param {JSON} monitorConfig Configuration object for the monitor - * @param {*} interval resource fetching interval + * @param {JSON} resourceMonitorOptions Configuration options for the monitor */ - constructor(monitorConfig, interval) { - super(monitorConfig, interval); + constructor(resourceMonitorOptions) { + super(resourceMonitorOptions); this.isReading = false; this.intervalObj = null; this.pids = {}; // pid history array - - /* this.stats : record statistics of each process - { - 'id' : { // 'command args' - 'mem_usage' : [], - 'cpu_percent' : [], - } - ..... - } - */ - this.stats = {'time': []}; - this.watchItems = []; - for(let i = 0 ; i < this.monitorConfig.processes.length ; i++) { - if(this.monitorConfig.processes[i].hasOwnProperty('command')) { - let id = this.getId(this.monitorConfig.processes[i]); - this.stats[id] = this.newStat(); - this.watchItems.push(this.monitorConfig.processes[i]); - } - } } /** @@ -202,7 +182,32 @@ class MonitorProcess extends MonitorInterface { * @async */ async start() { + // Configure items to be recorded + this.stats = {'time': []}; + this.watchItems = []; + + + /* this.stats : record statistics of each process + { + 'id' : { // 'command args' + 'mem_usage' : [], + 'cpu_percent' : [], + } + ..... + } + */ + for (let i = 0 ; i < this.options.processes.length ; i++) { + if (this.options.processes[i].hasOwnProperty('command')) { + let id = this.getId(this.options.processes[i]); + Logger.info(`Registering ${id} within process monitor`); + this.stats[id] = this.newStat(); + this.watchItems.push(this.options.processes[i]); + } + } + + // First read await this.readStats(); + // Start interval monitor const self = this; this.intervalObj = setInterval(async () => { await self.readStats(); } , this.interval); @@ -214,22 +219,7 @@ class MonitorProcess extends MonitorInterface { * @async */ async restart() { - clearInterval(this.intervalObj); - for (let key in this.stats) { - if (key === 'time') { - this.stats[key] = []; - } else { - for (let v in this.stats[key]) { - this.stats[key][v] = []; - } - } - } - - for (let key in this.pids) { - usage.unmonitor(key); - } - this.pids = []; - + await this.stop(); await this.start(); } @@ -242,7 +232,7 @@ class MonitorProcess extends MonitorInterface { this.containers = []; this.stats = {'time': []}; - for(let key in this.pids) { + for (let key in this.pids) { usage.unmonitor(key); } this.pids = []; @@ -302,7 +292,7 @@ class MonitorProcess extends MonitorInterface { } // Retrieve Chart data - const chartTypes = this.monitorConfig.charting; + const chartTypes = this.options.charting; let chartStats = []; if (chartTypes) { chartStats = ChartBuilder.retrieveChartStats(this.constructor.name, chartTypes, testLabel, resourceStats); diff --git a/packages/caliper-core/lib/manager/monitors/monitor-prometheus.js b/packages/caliper-core/lib/manager/monitors/monitor-prometheus.js index 19ec03143e..ffb4c35779 100644 --- a/packages/caliper-core/lib/manager/monitors/monitor-prometheus.js +++ b/packages/caliper-core/lib/manager/monitors/monitor-prometheus.js @@ -19,7 +19,6 @@ const ConfigUtil = require('../../common/config/config-util'); const ChartBuilder = require('../charts/chart-builder'); const Logger = Util.getLogger('monitor-prometheus'); const MonitorInterface = require('./monitor-interface'); -const PrometheusPushClient = require('../../common/prometheus/prometheus-push-client'); const PrometheusQueryClient = require('../../common/prometheus/prometheus-query-client'); const PrometheusQueryHelper = require('../../common/prometheus/prometheus-query-helper'); @@ -30,26 +29,24 @@ class PrometheusMonitor extends MonitorInterface { /** * Constructor - * @param {JSON} monitorConfig Monitor config information - * @param {*} interval resource fetching interval + * @param {JSON} resourceMonitorOptions Configuration options for the monitor */ - constructor(monitorConfig, interval) { - super(monitorConfig, interval); + constructor(resourceMonitorOptions) { + super(resourceMonitorOptions); this.precision = ConfigUtil.get(ConfigUtil.keys.Report.Precision, 3); - this.prometheusPushClient = new PrometheusPushClient(monitorConfig.push_url); - this.prometheusQueryClient = new PrometheusQueryClient(monitorConfig.url); + this.prometheusQueryClient = new PrometheusQueryClient(this.options.url); // User defined options for monitoring - if (monitorConfig.hasOwnProperty('metrics')) { + if (this.options.hasOwnProperty('metrics')) { // Might have an ignore list - if (monitorConfig.metrics.hasOwnProperty('ignore')) { - this.ignore = monitorConfig.metrics.ignore; + if (this.options.metrics.hasOwnProperty('ignore')) { + this.ignore = this.options.metrics.ignore; } else { Logger.info('No monitor metrics `ignore` option specified, will provide statistics on all items retrieved by queries'); } // Might have user specified queries to run - if (monitorConfig.metrics.hasOwnProperty('include')) { - this.include = monitorConfig.metrics.include; + if (this.options.metrics.hasOwnProperty('include')) { + this.include = this.options.metrics.include; } else { Logger.info('No monitor metrics `include` options specified, unable to provide statistics on any resources'); } @@ -58,14 +55,6 @@ class PrometheusMonitor extends MonitorInterface { } } - /** - * Retrieve the push client - * @returns {PrometheusPushClient} the push client - */ - getPushClient(){ - return this.prometheusPushClient; - } - /** * Retrieve the query client * @returns {PrometheusQueryClient} the query client @@ -74,14 +63,6 @@ class PrometheusMonitor extends MonitorInterface { return this.prometheusQueryClient; } - /** - * Retrieve the PushGateway URL from the monitor config - * @returns{String} the PushGateway URL - */ - getPushGatewayURL(){ - return this.monitorConfig.push_url; - } - /** * start monitoring - reset the initial query time index * @async @@ -169,11 +150,13 @@ class PrometheusMonitor extends MonitorInterface { metricArray.push(metricMap); } } - chartArray.push(metricArray); + if (metricArray.length > 0) { + chartArray.push(metricArray); + } } // Retrieve Chart data - const chartTypes = this.monitorConfig.charting; + const chartTypes = this.options.charting; const chartStats = []; if (chartTypes) { for (const metrics of chartArray) { diff --git a/packages/caliper-core/lib/manager/orchestrators/monitor-orchestrator.js b/packages/caliper-core/lib/manager/orchestrators/monitor-orchestrator.js index e9d4a4fd8a..00df7879c2 100644 --- a/packages/caliper-core/lib/manager/orchestrators/monitor-orchestrator.js +++ b/packages/caliper-core/lib/manager/orchestrators/monitor-orchestrator.js @@ -39,34 +39,30 @@ class MonitorOrchestrator { this.started = false; this.monitors = new Map(); // Parse the config and retrieve the monitor types - const monitorConfig = benchmarkConfig.monitor; - if(typeof monitorConfig === 'undefined') { - logger.info('No monitor specified, will default to "none"'); + const resourceMonitors = benchmarkConfig.monitors && benchmarkConfig.monitors.resource ? benchmarkConfig.monitors.resource : []; + if (resourceMonitors.length === 0) { + logger.info('No resource monitors specified'); return; } - if(typeof monitorConfig.type === 'undefined') { - throw new Error('Failed to find monitor types in config file'); - } - - let monitorTypes = Array.isArray(monitorConfig.type) ? monitorConfig.type : [monitorConfig.type]; - monitorTypes = Array.from(new Set(monitorTypes)); // remove duplicates - for (let type of monitorTypes) { + for (const resourceMonitor of resourceMonitors) { + const monitorModule = resourceMonitor.module; let monitor = null; - if(type === DOCKER) { - monitor = new DockerMonitor(monitorConfig.docker, monitorConfig.interval); - } else if(type === PROCESS) { - monitor = new ProcessMonitor(monitorConfig.process, monitorConfig.interval); - } else if(type === PROMETHEUS) { - monitor = new PrometheusMonitor(monitorConfig.prometheus, monitorConfig.interval); - } else if(type === NONE) { + logger.info(`Attempting to create resource monitor of type ${monitorModule}`); + if (monitorModule === DOCKER) { + monitor = new DockerMonitor(resourceMonitor.options); + } else if(monitorModule === PROCESS) { + monitor = new ProcessMonitor(resourceMonitor.options); + } else if(monitorModule === PROMETHEUS) { + monitor = new PrometheusMonitor(resourceMonitor.options); + } else if(monitorModule === NONE) { continue; } else { - const msg = `Unsupported monitor type ${type}, must be one of ${VALID_MONITORS}`; + const msg = `Unsupported monitor type ${monitorModule}, must be one of ${VALID_MONITORS}`; logger.error(msg); throw new Error(msg); } - this.monitors.set(type, monitor); + this.monitors.set(monitorModule, monitor); } } diff --git a/packages/caliper-core/lib/manager/orchestrators/round-orchestrator.js b/packages/caliper-core/lib/manager/orchestrators/round-orchestrator.js index 686d69e406..6a2b6b58a5 100644 --- a/packages/caliper-core/lib/manager/orchestrators/round-orchestrator.js +++ b/packages/caliper-core/lib/manager/orchestrators/round-orchestrator.js @@ -40,7 +40,7 @@ class RoundOrchestrator { this.workerOrchestrator = new WorkerOrchestrator(this.benchmarkConfig, workerArguments); this.monitorOrchestrator = new MonitorOrchestrator(this.benchmarkConfig); this.report = new Report(this.monitorOrchestrator, this.benchmarkConfig, this.networkConfig); - this.testObserver = new TestObserver(this.benchmarkConfig); + this.testObserver = new TestObserver(); } /** @@ -152,8 +152,7 @@ class RoundOrchestrator { rateControl: round.rateControl, trim: round.trim || 0, workload: round.workload, - testRound: index, - pushUrl: this.monitorOrchestrator.hasMonitor('prometheus') ? this.monitorOrchestrator.getMonitor('prometheus').getPushGatewayURL() : undefined + testRound: index }; if (round.txNumber) { @@ -170,15 +169,7 @@ class RoundOrchestrator { this.report.createReport(); - // Start all the monitors - try { - await this.monitorOrchestrator.startAllMonitors(); - logger.info('Monitors successfully started'); - } catch (err) { - logger.error(`Could not start monitors: ${err.stack || err}`); - } - - // Start all the monitors + // Prepare worker connections try { logger.info('Preparing worker connections'); await this.workerOrchestrator.prepareWorkerConnections(); @@ -198,30 +189,34 @@ class RoundOrchestrator { await this.workerOrchestrator.prepareTestRound(roundConfig); // Run main test round + // - Start observer and monitors so that only the main round is considered in retrieved metrics this.testObserver.startWatch(this.workerOrchestrator); + try { + await this.monitorOrchestrator.startAllMonitors(); + logger.info('Monitors successfully started'); + } catch (err) { + logger.error(`Could not start monitors: ${err.stack || err}`); + } + // - Run main test const {results, start, end} = await this.workerOrchestrator.startTest(roundConfig); await this.testObserver.stopWatch(); // Build the report // - TPS - let idx; - if (this.monitorOrchestrator.hasMonitor('prometheus')) { - idx = await this.report.processPrometheusTPSResults({start, end}, roundConfig, index); - } else { - idx = await this.report.processLocalTPSResults(results, roundConfig); - } - + const idx = await this.report.processTPSResults(results, roundConfig); // - Resource utilization await this.report.buildRoundResourceStatistics(idx, roundConfig.label); success++; - logger.info(`Finished round ${index + 1} (${roundConfig.label}) in ${(end - start) / 1000.0} seconds`); + logger.info(`Finished round ${index + 1} (${roundConfig.label}) in ${CaliperUtils.millisToSeconds(end - start)} seconds`); + + // Stop all monitors so that they can be reset between rounds + await this.monitorOrchestrator.stopAllMonitors(); // sleep some between the rounds if (index !== roundConfigs.length - 1) { logger.info('Waiting 5 seconds for the next round...'); await CaliperUtils.sleep(5000); - await this.monitorOrchestrator.restartAllMonitors(); } } catch (err) { await this.testObserver.stopWatch(); diff --git a/packages/caliper-core/lib/manager/orchestrators/worker-orchestrator.js b/packages/caliper-core/lib/manager/orchestrators/worker-orchestrator.js index fafe99f7c5..548c8c4f27 100644 --- a/packages/caliper-core/lib/manager/orchestrators/worker-orchestrator.js +++ b/packages/caliper-core/lib/manager/orchestrators/worker-orchestrator.js @@ -22,6 +22,7 @@ const ConfigUtils = require('../../common/config/config-util'); const Constants = require('./../../common/utils/constants'); const MessageTypes = require('./../../common/utils/constants').Messages.Types; +const TransactionStatisticsCollector = require('./../../common/core/transaction-statistics-collector'); const RegisterMessage = require('./../../common/messages/registerMessage'); const AssignIdMessage = require('./../../common/messages/assignIdMessage'); @@ -557,34 +558,28 @@ class WorkerOrchestrator { } /** - * Format the final test results for subsequent consumption from [ {result: [], start: val, end: val}, {result: [], start: val, end: val}, {result: [], start: val, end: val}] - * to {results: [val, val], start: val, end: val} - * @param {JSON[]} results an Array of JSON objects + * Format the final test results for subsequent consumption by the round orchestrator + * to {results: TransactionStatisticsCollector, start: val, end: val} + * @param {TransactionStatisticsCollector[]} workerResults an Array of TransactionStatisticsCollector objects * @return {JSON} an appropriately formatted result */ - formatResults(results) { - - let resultArray = []; - let allStartedTime = null; - let allFinishedTime = null; - for (const workerResult of results){ - // Start building the array of all worker results - resultArray = resultArray.concat(workerResult.results); - - // Track all started/complete times - if (!allStartedTime || workerResult.start > allStartedTime) { - allStartedTime = workerResult.start; - } - - if (!allFinishedTime || workerResult.end < allFinishedTime) { - allFinishedTime = workerResult.end; - } + formatResults(workerResults) { + logger.debug(`Entering formatResults with ${JSON.stringify(workerResults)}`); + + const txnCollectorArray = []; + for (const workerResult of workerResults) { + // Start building the array of all worker stats + const stats = workerResult.stats; + const txnCollector = TransactionStatisticsCollector.loadFromObject(stats); + txnCollectorArray.push(txnCollector); } + const results = TransactionStatisticsCollector.mergeCollectorResults(txnCollectorArray); + return { - results: resultArray, - start: allStartedTime, - end: allFinishedTime + results, + start: results.getRoundStartTime(), + end: results.getRoundFinishTime() }; } diff --git a/packages/caliper-core/lib/manager/report/report.js b/packages/caliper-core/lib/manager/report/report.js index 1208ea9957..f1bce823d4 100644 --- a/packages/caliper-core/lib/manager/report/report.js +++ b/packages/caliper-core/lib/manager/report/report.js @@ -15,9 +15,7 @@ 'use strict'; const ReportBuilder = require('./report-builder'); -const PrometheusQueryHelper = require('../../common/prometheus/prometheus-query-helper'); const CaliperUtils = require('../../common/utils/caliper-utils'); -const TransactionStatistics = require('../../common/core/transaction-statistics'); const Logger = CaliperUtils.getLogger('report-builder'); const table = require('table'); @@ -149,95 +147,25 @@ class Report { /** * Create a result map from locally gathered values * @param {string} testLabel the test label name - * @param {JSON} results txStatistics JSON object + * @param {TransactionStatisticsCollector} results TransactionStatisticsCollector containing cumulative worker transaction statistics * @return {Map} a Map of key value pairing to create the default result table */ - getLocalResultValues(testLabel, results) { + getResultValues(testLabel, results) { Logger.debug ('getLocalResultValues called with: ', JSON.stringify(results)); const resultMap = this.getResultColumnMap(); resultMap.set('Name', testLabel ? testLabel : 'unknown'); - resultMap.set('Succ', results.hasOwnProperty('succ') ? results.succ : '-'); - resultMap.set('Fail', results.hasOwnProperty('fail') ? results.fail : '-'); - resultMap.set('Max Latency (s)', (results.hasOwnProperty('delay') && results.delay.hasOwnProperty('max')) ? results.delay.max.toFixed(2) : '-'); - resultMap.set('Min Latency (s)', (results.hasOwnProperty('delay') && results.delay.hasOwnProperty('min')) ? results.delay.min.toFixed(2) : '-'); - resultMap.set('Avg Latency (s)', (results.hasOwnProperty('delay') && results.delay.hasOwnProperty('sum') && results.hasOwnProperty('succ')) ? (results.delay.sum / results.succ).toFixed(2) : '-'); - - // Send rate needs a little more conditioning than sensible for a ternary op - if (results.hasOwnProperty('succ') && results.hasOwnProperty('fail') && results.hasOwnProperty('create') && results.create.hasOwnProperty('max') && results.create.hasOwnProperty('min')) { - const sendRate = (results.create.max === results.create.min) ? (results.succ + results.fail) : ((results.succ + results.fail) / (results.create.max - results.create.min)).toFixed(1); - resultMap.set('Send Rate (TPS)', sendRate); - } else { - resultMap.set('Send Rate (TPS)', '-'); - } - - // Observed TPS needs a little more conditioning than sensible for a ternary op - if (results.hasOwnProperty('succ') && results.hasOwnProperty('final') && results.final.hasOwnProperty('last') && results.hasOwnProperty('create') && results.create.hasOwnProperty('min')) { - const tps = (results.final.last === results.create.min) ? results.succ : (results.succ / (results.final.last - results.create.min)).toFixed(1); - resultMap.set('Throughput (TPS)', tps); - } else { - resultMap.set('Throughput (TPS)', '-'); - } - - return resultMap; - } - - /** - * Create a result mapping through querying a Prometheus server - * @param {string} testLabel the test label name - * @param {string} round the test round - * @param {number} startTime start time for Prometheus data query, in milliseconds since epoch - * @param {number} endTime end time for Prometheus data query, in milliseconds since epoch - * @return {Map} a Map of key value pairing to create the result table - */ - async getPrometheusResultValues(testLabel, round, startTime, endTime) { - const startQuery = startTime/1000; //convert to seconds - const endQuery = endTime/1000; - - const resultMap = this.getResultColumnMap(); - resultMap.set('Name', testLabel ? testLabel : 'unknown'); - - // Successful transactions - const txSuccessQuery = `sum(caliper_txn_success{instance="${testLabel}", round="${round}"})`; - const txSuccessCountResponse = await this.queryClient.query(txSuccessQuery, endQuery); - const txSuccessCount = txSuccessCountResponse ? PrometheusQueryHelper.extractFirstValueFromQueryResponse(txSuccessCountResponse) : '-'; - resultMap.set('Succ', txSuccessCount); - - // Failed transactions - const txFailQuery = `sum(caliper_txn_failure{instance="${testLabel}", round="${round}"})`; - const txFailCountResponse = await this.queryClient.query(txFailQuery, endQuery); - const txFailCount = txFailCountResponse ? PrometheusQueryHelper.extractFirstValueFromQueryResponse(txFailCountResponse) : '-'; - resultMap.set('Fail', txFailCount); - - // Maximum latency - const maxLatencyQuery = `max(caliper_latency{instance="${testLabel}", round="${round}"})`; - const maxLatenciesResponse = await this.queryClient.rangeQuery(maxLatencyQuery, startQuery, endQuery); - const maxLatenciesStats = PrometheusQueryHelper.extractStatisticFromRange(maxLatenciesResponse, 'max'); - const maxLatency = maxLatenciesStats.has('unknown') ? maxLatenciesStats.get('unknown').toFixed(2) : '-'; - resultMap.set('Max Latency (s)', maxLatency); - - // Min latency - const minLatencyQuery = `min(caliper_latency{instance="${testLabel}", round="${round}"})`; - const minLatenciesResponse = await this.queryClient.rangeQuery(minLatencyQuery, startQuery, endQuery); - const minLatenciesStats = PrometheusQueryHelper.extractStatisticFromRange(minLatenciesResponse, 'min'); - const minLatency = minLatenciesStats.has('unknown') ? minLatenciesStats.get('unknown').toFixed(2) : '-'; - resultMap.set('Min Latency (s)', minLatency); - - // Avg Latency - const avgLatencyQuery = `avg(caliper_latency{instance="${testLabel}", round="${round}"})`; - const avgLatenciesResponse = await this.queryClient.rangeQuery(avgLatencyQuery, startQuery, endQuery); - const avgLatenciesStats = PrometheusQueryHelper.extractStatisticFromRange(avgLatenciesResponse, 'avg'); - const avgLatency = avgLatenciesStats.has('unknown') ? avgLatenciesStats.get('unknown').toFixed(2) : '-'; - resultMap.set('Avg Latency (s)', avgLatency); - - // Avg Submit Rate within the time bounding - const avgSubmitRateQuery = `sum(caliper_txn_submit_rate{instance="${testLabel}", round="${round}"})`; - const avgSubmitRateResponse = await this.queryClient.rangeQuery(avgSubmitRateQuery, startQuery, endQuery); - const avgSubmitRateStats = PrometheusQueryHelper.extractStatisticFromRange(avgSubmitRateResponse, 'avg'); - const avgSubmitRate = avgSubmitRateStats.has('unknown') ? avgSubmitRateStats.get('unknown').toFixed(2) : '-'; - resultMap.set('Send Rate (TPS)', avgSubmitRate); - - // Average TPS (completed transactions) - const tps = ((txSuccessCount + txFailCount)/(endQuery - startQuery)).toFixed(1); + resultMap.set('Succ', results.getTotalSuccessfulTx()); + resultMap.set('Fail', results.getTotalFailedTx()); + resultMap.set('Max Latency (s)', CaliperUtils.millisToSeconds(results.getMaxLatencyForSuccessful()).toFixed(2)); + resultMap.set('Min Latency (s)', CaliperUtils.millisToSeconds(results.getMinLatencyForSuccessful()).toFixed(2)); + resultMap.set('Avg Latency (s)', results.getTotalSuccessfulTx() > 0 ? (CaliperUtils.millisToSeconds(results.getTotalLatencyForSuccessful() / results.getTotalSuccessfulTx())).toFixed(2) : '-'); + + // Send rate + const sendRate = ((results.getTotalSuccessfulTx() + results.getTotalFailedTx()) / (CaliperUtils.millisToSeconds(results.getLastCreateTime() - results.getFirstCreateTime()))).toFixed(1); + resultMap.set('Send Rate (TPS)', sendRate); + + // Observed TPS + const tps = ((results.getTotalSuccessfulTx() + results.getTotalFailedTx()) / (CaliperUtils.millisToSeconds(results.getLastFinishTime() - results.getFirstCreateTime()))).toFixed(1); resultMap.set('Throughput (TPS)', tps); return resultMap; @@ -256,21 +184,13 @@ class Report { /** * Augment the report with details generated through extraction from local process reporting - * @param {JSON} results JSON object {tsStats: txStats[], start: number, end: number} containing an array of txStatistics + * @param {TransactionStatisticsCollector} results cumulative results from all worker TransactionStatisticsCollectors * @param {Object} roundConfig test round configuration object * @return {Promise} promise object containing the current report index */ - async processLocalTPSResults(results, roundConfig){ + async processTPSResults(results, roundConfig) { try { - let resultSet; - - if (TransactionStatistics.mergeDefaultTxStats(results) === 0) { - resultSet = TransactionStatistics.createNullDefaultTxStats(); - } else { - resultSet = results[0]; - } - - const resultMap = this.getLocalResultValues(roundConfig.label, resultSet); + const resultMap = this.getResultValues(roundConfig.label, results); // Add this set of results to the main round collection this.resultsByRound.push(resultMap); @@ -285,7 +205,7 @@ class Report { return idx; } catch(error) { - Logger.error(`processLocalTPSResults failed with error: ${error}`); + Logger.error(`processTPSResults failed with error: ${error}`); throw error; } } @@ -309,38 +229,6 @@ class Report { } } - /** - * Augment the report with details generated through extraction from Prometheus - * @param {JSON} timing JSON object containing start/end times required to query - * the prometheus server - * @param {Object} roundConfig test round configuration object - * @param {number} round the current test round - * @return {Promise} promise object containing the report index - * @async - */ - async processPrometheusTPSResults(timing, roundConfig, round){ - try { - const resultMap = await this.getPrometheusResultValues(roundConfig.label, round, timing.start, timing.end); - - // Add this set of results to the main round collection - this.resultsByRound.push(resultMap); - - // Print TPS result for round to console - Logger.info('### Test result ###'); - const tableArray = this.convertToTable(resultMap); - this.printTable(tableArray); - - // Add TPS to the report - let idx = this.reportBuilder.addBenchmarkRound(roundConfig); - this.reportBuilder.setRoundPerformance(roundConfig.label, idx, tableArray); - - return idx; - } catch (error) { - Logger.error(`processPrometheusTPSResults failed with error: ${error}`); - throw error; - } - } - /** * Print the generated report to file * @async diff --git a/packages/caliper-core/lib/manager/test-observers/local-observer.js b/packages/caliper-core/lib/manager/test-observers/default-observer.js similarity index 57% rename from packages/caliper-core/lib/manager/test-observers/local-observer.js rename to packages/caliper-core/lib/manager/test-observers/default-observer.js index f6503ac735..95e014f850 100644 --- a/packages/caliper-core/lib/manager/test-observers/local-observer.js +++ b/packages/caliper-core/lib/manager/test-observers/default-observer.js @@ -16,25 +16,25 @@ 'use strict'; const TestObserverInterface = require('./observer-interface'); +const TransactionStatisticsCollector = require('../../common/core/transaction-statistics-collector'); const Utils = require('../../common/utils/caliper-utils'); -const Logger = Utils.getLogger('local-observer'); +const ConfigUtil = require('../../common/config/config-util'); +const Logger = Utils.getLogger('default-observer'); /** - * LocalObserver class used to observe test statistics via terminal + * DefaultObserver class used to observe test statistics via terminal */ -class LocalObserver extends TestObserverInterface { +class DefaultObserver extends TestObserverInterface { /** * Constructor - * @param {object} benchmarkConfig The benchmark configuration object. */ - constructor(benchmarkConfig) { - super(benchmarkConfig); + constructor() { + super(); // set the observer interval - const interval = (this.benchmarkConfig.observer && this.benchmarkConfig.observer.interval) ? this.benchmarkConfig.observer.interval : 1; - this.observeInterval = interval * 1000; - Logger.info(`Observer interval set to ${interval} seconds`); + this.observeInterval = ConfigUtil.get(ConfigUtil.keys.Progress.Reporting.Interval); + Logger.info(`Observer interval set to ${this.observeInterval} seconds`); this.observeIntervalObject = null; this.updateTail = 0; this.updateID = 0; @@ -68,23 +68,10 @@ class LocalObserver extends TestObserverInterface { * @param {*} suc successful * @param {*} fail fail */ - addThroughput(sub, suc, fail) { - this.testData.throughput.submitted.push(sub/this.observeInterval); - this.testData.throughput.succeeded.push(suc/this.observeInterval); - this.testData.throughput.failed.push(fail/this.observeInterval); - if (this.testData.throughput.x.length < this.testData.throughput.submitted.length) { - let last = this.testData.throughput.x[this.testData.throughput.x.length - 1]; - this.testData.throughput.x.push(last + this.observeInterval); - } - if (this.testData.throughput.submitted.length > this.testData.maxlen) { - this.testData.throughput.submitted.shift(); - this.testData.throughput.succeeded.shift(); - this.testData.throughput.failed.shift(); - this.testData.throughput.x.shift(); - } - this.testData.summary.txSub += sub; - this.testData.summary.txSucc += suc; - this.testData.summary.txFail += fail; + setThroughput(sub, suc, fail) { + this.testData.summary.txSub = sub; + this.testData.summary.txSucc = suc; + this.testData.summary.txFail = fail; } /** @@ -97,7 +84,7 @@ class LocalObserver extends TestObserverInterface { this.testData.latency.max.push(max); this.testData.latency.min.push(min); this.testData.latency.avg.push(avg); - if(this.testData.latency.x.length < this.testData.latency.max.length) { + if (this.testData.latency.x.length < this.testData.latency.max.length) { let last = this.testData.latency.x[this.testData.latency.x.length - 1]; this.testData.latency.x.push(last + this.observeInterval); } @@ -113,12 +100,6 @@ class LocalObserver extends TestObserverInterface { * Reset test data */ resetTestData() { - this.testData.throughput = { - x: [], - submitted: [0], - succeeded: [0], - failed: [0] - }; this.testData.latency = { x: [], max: [0], @@ -139,47 +120,64 @@ class LocalObserver extends TestObserverInterface { */ refreshData(updates) { if (updates.length === 0 || Object.entries(updates[0]).length === 0) { - this.addThroughput(0,0,0); + // Nothing to update with, set zero + this.setThroughput(0,0,0); this.addLatency(0,0,0); } else { - let sub = 0, suc = 0, fail = 0; let deMax = -1, deMin = -1, deAvg = 0; - for(let i = 0 ; i < updates.length ; i++) { - let data = updates[i]; + + // Updates may come from multiple workers, and may get more than one update per worker in an interval + // - We need to sum the transaction counts, and evaluate the min/max/average of latencies + const txnCollectionMap = new Map(); + for (let i = 0 ; i < updates.length ; i++) { + const data = updates[i]; if (data.type.localeCompare('txReset') === 0) { - // Resetting values + // Resetting internal store + // - return once reset to prevent printing unrequired values Logger.info('Resetting txCount indicator count'); this.resetTestData(); - continue; + return; } - sub += data.submitted; - suc += data.committed.succ; - fail += data.committed.fail; - if(data.committed.succ > 0) { - if(deMax === -1 || deMax < data.committed.delay.max) { - deMax = data.committed.delay.max; + // Work on the stats object + const stats = data.stats; + const txnCollector = TransactionStatisticsCollector.loadFromObject(stats); + const workerIndex = txnCollector.getWorkerIndex(); + + txnCollectionMap.set(workerIndex, txnCollector); + } + + if (txnCollectionMap.size > 0) { + let txnCollectorArray = Array.from( txnCollectionMap.values() ); + const txnCollection = TransactionStatisticsCollector.mergeCollectorResults(txnCollectorArray); + + // Base transaction counters + this.setThroughput(txnCollection.getTotalSubmittedTx(), txnCollection.getTotalSuccessfulTx(), txnCollection.getTotalFailedTx()); + + // Latencies calculated on successful transactions + if (txnCollection.getTotalSuccessfulTx() > 0) { + if (deMax === -1 || deMax < txnCollection.getMaxLatencyForSuccessful()) { + deMax = txnCollection.getMaxLatencyForSuccessful(); } - if(deMin === -1 || deMin > data.committed.delay.min) { - deMin = data.committed.delay.min; + if (deMin === -1 || deMin > txnCollection.getMinLatencyForSuccessful()) { + deMin = txnCollection.getMinLatencyForSuccessful(); } - deAvg += data.committed.delay.sum; + deAvg += txnCollection.getTotalLatencyForSuccessful(); + } + + if (txnCollection.getTotalSuccessfulTx() > 0) { + deAvg /= txnCollection.getTotalSuccessfulTx(); } } - if(suc > 0) { - deAvg /= suc; - } - this.addThroughput(sub, suc, fail); - if(isNaN(deMax) || isNaN(deMin) || deAvg === 0) { + if (isNaN(deMax) || isNaN(deMin) || deAvg === 0) { this.addLatency(0,0,0); - } - else { + } else { this.addLatency(deMax, deMin, deAvg); } - } + // Log current statistics using global observer store as above update might not have occurred Logger.info('[' + this.testName + ' Round ' + this.testRound + ' Transaction Info] - Submitted: ' + this.testData.summary.txSub + ' Succ: ' + this.testData.summary.txSucc + ' Fail:' + this.testData.summary.txFail + @@ -191,18 +189,19 @@ class LocalObserver extends TestObserverInterface { * @async */ async update() { - if (typeof this.clientOrchestrator === 'undefined') { + if (typeof this.workerOrchestrator === 'undefined') { this.refreshData([]); return; } - let updates = this.clientOrchestrator.getUpdates(); - if(updates.id > this.updateID) { // new buffer + let updates = this.workerOrchestrator.getUpdates(); + if (updates.id > this.updateID) { + // new buffer this.updateTail = 0; - this.updateID = updates.id; + this.updateID = updates.id; } let data = []; - let len = updates.data.length; - if(len > this.updateTail) { + let len = updates.data.length; + if (len > this.updateTail) { data = updates.data.slice(this.updateTail, len); this.updateTail = len; } @@ -211,10 +210,10 @@ class LocalObserver extends TestObserverInterface { /** * Start watching the test output of the orchestrator - * @param {ClientOrchestrator} clientOrchestrator the client orchestrator + * @param {WorkerOrchestrator} workerOrchestrator the worker orchestrator */ - startWatch(clientOrchestrator) { - this.clientOrchestrator = clientOrchestrator; + startWatch(workerOrchestrator) { + this.workerOrchestrator = workerOrchestrator; if(this.observeIntervalObject === null) { this.updateTail = 0; this.updateID = 0; @@ -244,6 +243,7 @@ class LocalObserver extends TestObserverInterface { setBenchmark(name) { this.testName = name; } + /** * Set the test round for the watcher * @param{*} roundIdx the round index @@ -251,15 +251,22 @@ class LocalObserver extends TestObserverInterface { setRound(roundIdx) { this.testRound = roundIdx; } + + /** + * Called when new TX stats are available. + * @param {TransactionStatisticsCollector} stats The TX stats collector instance. + */ + txUpdateArrived(stats) { + // TODO: push model instead of pull + } } /** - * Creates a new LocalObserver instance. - * @param {object} benchmarkConfig The benchmark configuration object. - * @return {TestObserverInterface} The LocalObserver instance. + * Creates a new DefaultObserver instance. + * @return {TestObserverInterface} The DefaultObserver instance. */ -function createTestObserver(benchmarkConfig) { - return new LocalObserver(benchmarkConfig); +function createTestObserver() { + return new DefaultObserver(); } module.exports.createTestObserver = createTestObserver; diff --git a/packages/caliper-core/lib/manager/test-observers/null-observer.js b/packages/caliper-core/lib/manager/test-observers/null-observer.js index 68c06cacb4..43c620ddcd 100644 --- a/packages/caliper-core/lib/manager/test-observers/null-observer.js +++ b/packages/caliper-core/lib/manager/test-observers/null-observer.js @@ -25,10 +25,9 @@ class NullObserver extends TestObserverInterface { /** * Constructor - * @param {object} benchmarkConfig The benchmark configuration object. */ - constructor(benchmarkConfig) { - super(benchmarkConfig); + constructor() { + super(); Logger.info('Configured "null" observer'); } @@ -63,6 +62,7 @@ class NullObserver extends TestObserverInterface { setBenchmark(name) { Logger.debug('No action taken by NullObserver on setBenchmark'); } + /** * Set the test round for the watcher * @param{*} roundIdx the round index @@ -71,15 +71,21 @@ class NullObserver extends TestObserverInterface { Logger.debug('No action taken by NullObserver on setRound'); } + /** + * Called when new TX stats are available. + * @param {TransactionStatisticsCollector} stats The TX stats collector instance. + */ + txUpdateArrived(stats) { + Logger.debug('No action taken by NullObserver on txUpdateArrived'); + } } /** * Creates a new NullObserver instance. - * @param {object} benchmarkConfig The benchmark configuration object. * @return {TestObserverInterface} The NullObserver instance. */ -function createTestObserver(benchmarkConfig) { - return new NullObserver(benchmarkConfig); +function createTestObserver() { + return new NullObserver(); } module.exports.createTestObserver = createTestObserver; diff --git a/packages/caliper-core/lib/manager/test-observers/observer-interface.js b/packages/caliper-core/lib/manager/test-observers/observer-interface.js index cfb9b08df2..5d8f943f8f 100644 --- a/packages/caliper-core/lib/manager/test-observers/observer-interface.js +++ b/packages/caliper-core/lib/manager/test-observers/observer-interface.js @@ -24,10 +24,8 @@ class TestObserverInterface { /** * Constructor - * @param {object} benchmarkConfig The benchmark configuration object. */ - constructor(benchmarkConfig) { - this.benchmarkConfig = benchmarkConfig; + constructor() { } /** @@ -81,6 +79,14 @@ class TestObserverInterface { this._throwNotImplementedError('setRound'); } + /** + * Called when new TX stats are available. + * @param {TransactionStatisticsCollector} stats The TX stats collector instance. + */ + txUpdateArrived(stats) { + this._throwNotImplementedError('txUpdateArrived'); + } + } module.exports = TestObserverInterface; diff --git a/packages/caliper-core/lib/manager/test-observers/prometheus-observer.js b/packages/caliper-core/lib/manager/test-observers/prometheus-observer.js deleted file mode 100644 index fe66ce58e0..0000000000 --- a/packages/caliper-core/lib/manager/test-observers/prometheus-observer.js +++ /dev/null @@ -1,126 +0,0 @@ -/* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - - -'use strict'; - -const TestObserverInterface = require('./observer-interface'); -const PrometheusQueryClient = require('../../common/prometheus/prometheus-query-client'); -const PrometheusQueryHelper = require('../../common/prometheus/prometheus-query-helper'); -const Utils = require('../../common/utils/caliper-utils'); -const Logger = Utils.getLogger('prometheus-observer'); - -/** - * PrometheusObserver class used to observe test statistics via terminal - */ -class PrometheusObserver extends TestObserverInterface { - - /** - * Constructor - * @param {object} benchmarkConfig The benchmark configuration object. - */ - constructor(benchmarkConfig) { - super(benchmarkConfig); - - // determine interval - const interval = (this.benchmarkConfig.observer && this.benchmarkConfig.observer.interval) ? this.benchmarkConfig.observer.interval : 1; - this.observeInterval = interval * 1000; - - // Define the query client - const queryUrl = this.benchmarkConfig.monitor.prometheus.url; - this.queryClient = new PrometheusQueryClient(queryUrl); - Logger.info(`Configured observer to query URL ${queryUrl} every ${interval} seconds`); - } - - /** - * Perform an update - */ - async update() { - // Update using Prometheus Query - // -Successful transactions - const txSuccessQuery = `sum(caliper_txn_success{instance="${this.testName}", round="${this.testRound}"})`; - const txSuccessCountResponse = await this.queryClient.query(txSuccessQuery, Date.now()/1000); - const txSuccessCount = txSuccessCountResponse ? PrometheusQueryHelper.extractFirstValueFromQueryResponse(txSuccessCountResponse) : '-'; - - // -Failed transactions - const txFailQuery = `sum(caliper_txn_failure{instance="${this.testName}", round="${this.testRound}"})`; - const txFailCountResponse = await this.queryClient.query(txFailQuery, Date.now()/1000); - const txFailCount = txFailCountResponse ? PrometheusQueryHelper.extractFirstValueFromQueryResponse(txFailCountResponse) : '-'; - - // -Pending transactions - const txPendingQuery = `sum(caliper_txn_pending{instance="${this.testName}", round="${this.testRound}"})`; - const txPendingCountResponse = await this.queryClient.query(txPendingQuery, Date.now()/1000); - const txPendingCount = txPendingCountResponse ? PrometheusQueryHelper.extractFirstValueFromQueryResponse(txPendingCountResponse) : '-'; - - // Could use query on submitted, but quicker to do addition here than use query service - Logger.info('[' + this.testName + ' Round ' + this.testRound + ' Transaction Info] - Submitted: ' + (txSuccessCount + txFailCount + txPendingCount) + - ' Succ: ' + txSuccessCount + - ' Fail:' + txFailCount + - ' Unfinished:' + txPendingCount); - } - - /** - * Start observing the test output - * @param {ClientOrchestrator} clientOrchestrator the client orchestrator - */ - startWatch(clientOrchestrator) { - Logger.info(`Starting observer cycle with interval ${this.observeInterval} ms`); - this.clientOrchestrator = clientOrchestrator; - if(!this.observeIntervalObject) { - // start an interval to query updates - const self = this; - this.observeIntervalObject = setInterval(async() => { await self.update(); }, this.observeInterval); - } - } - - /** - * Stop watching the test output - * @async - */ - async stopWatch() { - if(this.observeIntervalObject) { - clearInterval(this.observeIntervalObject); - this.observeIntervalObject = null; - await Utils.sleep(this.observeInterval); - await this.update(); - } - } - - /** - * Set the test name to be reported - * @param {String} name the benchmark name - */ - setBenchmark(name) { - this.testName = name; - } - /** - * Set the test round for the watcher - * @param{*} roundIdx the round index - */ - setRound(roundIdx) { - this.testRound = roundIdx; - } - -} - -/** - * Creates a new PrometheusObserver instance. - * @param {object} benchmarkConfig The benchmark configuration object. - * @return {TestObserverInterface} The PrometheusObserver instance. - */ -function createTestObserver(benchmarkConfig) { - return new PrometheusObserver(benchmarkConfig); -} - -module.exports.createTestObserver = createTestObserver; diff --git a/packages/caliper-core/lib/manager/test-observers/test-observer.js b/packages/caliper-core/lib/manager/test-observers/test-observer.js index f58f6cbbf0..2c0e95466f 100644 --- a/packages/caliper-core/lib/manager/test-observers/test-observer.js +++ b/packages/caliper-core/lib/manager/test-observers/test-observer.js @@ -14,23 +14,22 @@ 'use strict'; const CaliperUtils = require('../../common/utils/caliper-utils'); +const ConfigUtil = require('../../common/config/config-util'); const Logger = CaliperUtils.getLogger('testObserver.js'); const builtInObservers = new Map([ ['none', './null-observer'], - ['local', './local-observer.js'], - ['prometheus', './prometheus-observer.js'] + ['default', './default-observer.js'] ]); const TestObserver = class { /** * Instantiates the proxy test observer and creates the configured observer behind it. - * @param {object} benchmarkConfig The benchmark configuration object. */ - constructor(benchmarkConfig) { - // Test observer is dynamically loaded, but defaults to none - const observerType = (benchmarkConfig.observer && benchmarkConfig.observer.type) ? benchmarkConfig.observer.type : 'none'; + constructor() { + // Test observer is dynamically loaded, based on progress reporting configuration + const observerType = ConfigUtil.get(ConfigUtil.keys.Progress.Reporting.Enabled) ? 'default' : 'none'; Logger.debug(`Creating test observer of type "${observerType}"`); @@ -43,7 +42,7 @@ const TestObserver = class { throw new Error(`${observerType} does not export the mandatory factory function 'createTestObserver'`); } - this.observer = factoryFunction(benchmarkConfig); + this.observer = factoryFunction(); } /** @@ -86,6 +85,14 @@ const TestObserver = class { this.observer.setRound(roundIdx); } + /** + * Called when new TX stats are available. + * @param {TransactionStatisticsCollector} stats The TX stats collector instance. + */ + txUpdateArrived(stats) { + this.observer.txUpdateArrived(stats); + } + }; module.exports = TestObserver; diff --git a/packages/caliper-core/lib/worker/caliper-worker.js b/packages/caliper-core/lib/worker/caliper-worker.js index 8443cc44f5..e6c2b03efa 100644 --- a/packages/caliper-core/lib/worker/caliper-worker.js +++ b/packages/caliper-core/lib/worker/caliper-worker.js @@ -14,16 +14,12 @@ 'use strict'; -const Config = require('../common/config/config-util.js'); const CaliperUtils = require('../common/utils/caliper-utils.js'); -const CircularArray = require('../common/utils/circular-array'); const RateControl = require('./rate-control/rateControl.js'); -const PrometheusClient = require('../common/prometheus/prometheus-push-client'); -const TransactionStatistics = require('../common/core/transaction-statistics'); -const TxResetMessage = require('./../common/messages/txResetMessage'); -const TxUpdateMessage = require('./../common/messages/txUpdateMessage'); const Events = require('../common/utils/constants').Events.Connector; +const TxObserverDispatch = require('./tx-observers/tx-observer-dispatch'); +const InternalTxObserver = require('./tx-observers/internal-tx-observer'); const Logger = CaliperUtils.getLogger('caliper-worker'); @@ -42,191 +38,28 @@ class CaliperWorker { constructor(connector, workerIndex, messenger, managerUuid) { this.connector = connector; this.workerIndex = workerIndex; - this.currentRoundIndex = -1; this.messenger = messenger; - this.managerUuid = managerUuid; - this.context = undefined; - this.txUpdateTime = Config.get(Config.keys.TxUpdateTime, 5000); - this.maxTxPromises = Config.get(Config.keys.Worker.MaxTxPromises, 100); - - // Internal stats - this.results = []; - this.txNum = 0; - this.txLastNum = 0; - this.resultStats = []; - this.trimType = 0; - this.trim = 0; - this.startTime = 0; - - // Prometheus related - this.prometheusClient = new PrometheusClient(); - this.totalTxCount = 0; - this.totalTxDelay = 0; - - /** - * The workload module instance associated with the current round, updated by {CaliperWorker.prepareTest}. - * @type {WorkloadModuleInterface} - */ - this.workloadModule = undefined; - const self = this; - this.connector.on(Events.TxsSubmitted, count => self.txNum += count); - this.connector.on(Events.TxsFinished, results => self.addResult(results)); - } - - /** - * Initialization update - */ - initUpdate() { - Logger.info('Initialization ongoing...'); - } - - /** - * Calculate real-time transaction statistics and send the txUpdated message - */ - txUpdate() { - let newNum = this.txNum - this.txLastNum; - this.txLastNum += newNum; - - // get a copy to work from - let newResults = this.results.slice(0); - this.results = []; - if (newResults.length === 0 && newNum === 0) { - return; - } - let newStats; - let publish = true; - if (newResults.length === 0) { - newStats = TransactionStatistics.createNullDefaultTxStats(); - publish = false; // no point publishing nothing!! - } else { - newStats = TransactionStatistics.getDefaultTxStats(newResults, false); - } - - // Update monitor - if (this.prometheusClient.gatewaySet() && publish){ - // Send to Prometheus push gateway - - // TPS and latency batch results for this current txUpdate limited set - const batchTxCount = newStats.succ + newStats.fail; - const batchTPS = (batchTxCount/this.txUpdateTime)*1000; // txUpdate is in ms - const batchLatency = newStats.delay.sum/batchTxCount; - this.prometheusClient.push('caliper_tps', batchTPS); - this.prometheusClient.push('caliper_latency', batchLatency); - this.prometheusClient.push('caliper_txn_submit_rate', (newNum/this.txUpdateTime)*1000); // txUpdate is in ms - - // Numbers for test round only - this.totalTxnSuccess += newStats.succ; - this.totalTxnFailure += newStats.fail; - this.prometheusClient.push('caliper_txn_success', this.totalTxnSuccess); - this.prometheusClient.push('caliper_txn_failure', this.totalTxnFailure); - this.prometheusClient.push('caliper_txn_pending', (this.txNum - (this.totalTxnSuccess + this.totalTxnFailure))); - } else { - // worker-orchestrator based update - // send(to, type, data) - const msg = new TxUpdateMessage(this.messenger.getUUID(), [this.managerUuid], {submitted: newNum, committed: newStats}); - this.messenger.send(msg); - } - - if (this.resultStats.length === 0) { - switch (this.trimType) { - case 0: // no trim - this.resultStats[0] = newStats; - break; - case 1: // based on duration - if (this.trim < (Date.now() - this.startTime)/1000) { - this.resultStats[0] = newStats; - } - break; - case 2: // based on number - if (this.trim < newResults.length) { - newResults = newResults.slice(this.trim); - newStats = TransactionStatistics.getDefaultTxStats(newResults, false); - this.resultStats[0] = newStats; - this.trim = 0; - } else { - this.trim -= newResults.length; - } - break; - } - } else { - this.resultStats[1] = newStats; - TransactionStatistics.mergeDefaultTxStats(this.resultStats); - } - } - - /** - * Method to reset values - */ - txReset(){ - - // Reset txn counters - this.results = []; - this.resultStats = []; - this.txNum = 0; - this.txLastNum = 0; - - if (this.prometheusClient.gatewaySet()) { - // Reset Prometheus - this.totalTxnSuccess = 0; - this.totalTxnFailure = 0; - this.prometheusClient.push('caliper_txn_success', 0); - this.prometheusClient.push('caliper_txn_failure', 0); - this.prometheusClient.push('caliper_txn_pending', 0); - } else { - // Reset Local - // send(to, type, data) - const msg = new TxResetMessage(this.messenger.getUUID(), [this.managerUuid]); - this.messenger.send(msg); - } - } + this.internalTxObserver = new InternalTxObserver(messenger, managerUuid); + this.txObserverDispatch = new TxObserverDispatch(messenger, this.internalTxObserver, managerUuid); - /** - * Add new test result into global array - * @param {Object} result test result, should be an array or a single JSON object - */ - addResult(result) { - if (Array.isArray(result)) { // contain multiple results - for(let i = 0 ; i < result.length ; i++) { - this.results.push(result[i]); - } - } else { - this.results.push(result); - } + // forward adapter notifications to the TX dispatch observer + const self = this; + this.connector.on(Events.TxsSubmitted, count => self.txObserverDispatch.txSubmitted(count)); + this.connector.on(Events.TxsFinished, results => self.txObserverDispatch.txFinished(results)); } /** - * Call before starting a new test - * @param {TestMessage} testMessage start test message + * Wait until every submitted TX is finished. + * @param {TransactionStatistics} roundStats The TX statistics of the current round. + * @private + * @async */ - beforeTest(testMessage) { - this.txReset(); - - // TODO: once prometheus is enabled, trim occurs as part of the retrieval query - // conditionally trim beginning and end results for this test run - if (testMessage.getTrimLength()) { - if (testMessage.getRoundDuration()) { - this.trimType = 1; - } else { - this.trimType = 2; - } - this.trim = testMessage.getTrimLength(); - } else { - this.trimType = 0; - } - - // Prometheus is specified if testMessage.pushUrl !== undefined - if (testMessage.getPrometheusPushGatewayUrl()) { - // - ensure counters reset - this.totalTxnSubmitted = 0; - this.totalTxnSuccess = 0; - this.totalTxnFailure = 0; - // - Ensure gateway base URL is set - if (!this.prometheusClient.gatewaySet()){ - this.prometheusClient.setGateway(testMessage.getPrometheusPushGatewayUrl()); - } - // - set target for this round test/round/worker - this.prometheusClient.configureTarget(testMessage.getRoundLabel(), testMessage.getRoundIndex(), this.workerIndex); + static async _waitForTxsToFinish(roundStats) { + // might lose some precision here, but checking the same after every TX result whether it was the last TX + // (so we could resolve a promise we're waiting for here) might hurt the sending rate + while (roundStats.getTotalFinishedTx() !== roundStats.getTotalSubmittedTx()) { + await CaliperUtils.sleep(100); } } @@ -246,177 +79,137 @@ class CaliperWorker { /** * Perform test with specified number of transactions + * @param {object} workloadModule The user test module. * @param {Object} number number of transactions to submit * @param {Object} rateController rate controller object * @async */ - async runFixedNumber(number, rateController) { - Logger.info(`Worker ${this.workerIndex} is starting TX number-based round ${this.currentRoundIndex + 1} (${number} TXs)`); - this.startTime = Date.now(); + async runFixedNumber(workloadModule, number, rateController) { + const stats = this.internalTxObserver.getCurrentStatistics(); + while (stats.getTotalSubmittedTx() < number) { + await rateController.applyRateControl(); - const circularArray = new CircularArray(this.maxTxPromises); - const self = this; - while (this.txNum < number) { // If this function calls this.workloadModule.submitTransaction() too quickly, micro task queue will be filled with unexecuted promises, // and I/O task(s) will get no chance to be execute and fall into starvation, for more detail info please visit: // https://snyk.io/blog/nodejs-how-even-quick-async-functions-can-block-the-event-loop-starve-io/ await this.setImmediatePromise(() => { - circularArray.add(self.workloadModule.submitTransaction()); + workloadModule.submitTransaction() + .catch(err => { Logger.error(`Unhandled error while executing TX: ${err.stack || err}`); }); }); - await rateController.applyRateControl(this.startTime, this.txNum, this.results, this.resultStats); } - await Promise.all(circularArray); - this.endTime = Date.now(); + await CaliperWorker._waitForTxsToFinish(stats); } /** * Perform test with specified test duration + * @param {object} workloadModule The user test module. * @param {Object} duration duration to run for * @param {Object} rateController rate controller object * @async */ - async runDuration(duration, rateController) { - Logger.info(`Worker ${this.workerIndex} is starting duration-based round ${this.currentRoundIndex + 1} (${duration} seconds)`); - this.startTime = Date.now(); + async runDuration(workloadModule, duration, rateController) { + const stats = this.internalTxObserver.getCurrentStatistics(); + let startTime = stats.getRoundStartTime(); + while ((Date.now() - startTime) < (duration * 1000)) { + await rateController.applyRateControl(); - // Use a circular array of Promises so that the Promise.all() call does not exceed the maximum permissable Array size - const circularArray = new CircularArray(this.maxTxPromises); - const self = this; - while ((Date.now() - this.startTime)/1000 < duration) { // If this function calls this.workloadModule.submitTransaction() too quickly, micro task queue will be filled with unexecuted promises, // and I/O task(s) will get no chance to be execute and fall into starvation, for more detail info please visit: // https://snyk.io/blog/nodejs-how-even-quick-async-functions-can-block-the-event-loop-starve-io/ await this.setImmediatePromise(() => { - circularArray.add(self.workloadModule.submitTransaction()); + workloadModule.submitTransaction() + .catch(err => { Logger.error(`Unhandled error while executing TX: ${err.stack || err}`); }); }); - await rateController.applyRateControl(this.startTime, this.txNum, this.results, this.resultStats); } - await Promise.all(circularArray); - this.endTime = Date.now(); + await CaliperWorker._waitForTxsToFinish(stats); } /** - * Clear the update interval - * @param {Object} txUpdateInter the test transaction update interval + * Prepare the round corresponding to the "prepare-round" message. + * @param {PrepareRoundMessage} prepareTestMessage The "execute-round" message containing schedule information. + * @return {Promise} The results of the round execution. */ - clearUpdateInter(txUpdateInter) { - // stop reporter - if (txUpdateInter) { - clearInterval(txUpdateInter); - txUpdateInter = null; - this.txUpdate(); - } - } + async prepareTest(prepareTestMessage) { + Logger.debug('Entering prepareTest'); - /** - * Perform test init within Benchmark - * @param {PrepareMessage} message the test details - * message = { - * label : label name, - * numb: total number of simulated txs, - * rateControl: rate controller to use - * trim: trim options - * workload: the workload object from the config, - * config: path of the blockchain config file - * totalClients = total number of clients, - * pushUrl = the url for the push gateway - * }; - * @async - */ - async prepareTest(message) { - Logger.debug('prepareTest() with:', message.stringify()); - this.currentRoundIndex = message.getRoundIndex(); + const roundIndex = prepareTestMessage.getRoundIndex(); - const workloadModuleFactory = CaliperUtils.loadModuleFunction(new Map(), message.getWorkloadSpec().module, 'createWorkloadModule'); + Logger.debug(`Worker #${this.workerIndex} creating workload module`); + const workloadModuleFactory = CaliperUtils.loadModuleFunction(new Map(), prepareTestMessage.getWorkloadSpec().module, 'createWorkloadModule'); this.workloadModule = workloadModuleFactory(); - const self = this; - let initUpdateInter = setInterval( () => { self.initUpdate(); } , self.txUpdateTime); - try { // Retrieve context for this round - this.context = await this.connector.getContext(message.getRoundIndex(), message.getWorkerArguments()); + const context = await this.connector.getContext(roundIndex, prepareTestMessage.getWorkerArguments()); // Run init phase of callback - Logger.info(`Info: worker ${this.workerIndex} prepare test phase for round ${this.currentRoundIndex + 1} is starting...`); - await this.workloadModule.initializeWorkloadModule(this.workerIndex, message.getWorkersNumber(), this.currentRoundIndex, message.getWorkloadSpec().arguments, this.connector, this.context); + Logger.info(`Info: worker ${this.workerIndex} prepare test phase for round ${roundIndex} is starting...`); + await this.workloadModule.initializeWorkloadModule(this.workerIndex, prepareTestMessage.getWorkersNumber(), roundIndex, prepareTestMessage.getWorkloadSpec().arguments, this.connector, context); await CaliperUtils.sleep(this.txUpdateTime); } catch (err) { - Logger.info(`Worker [${this.workerIndex}] encountered an error during prepare test phase for round ${this.currentRoundIndex + 1}: ${(err.stack ? err.stack : err)}`); + Logger.info(`Worker [${this.workerIndex}] encountered an error during prepare test phase for round ${roundIndex}: ${(err.stack ? err.stack : err)}`); throw err; } finally { - clearInterval(initUpdateInter); - Logger.info(`Info: worker ${this.workerIndex} prepare test phase for round ${this.currentRoundIndex + 1} is completed`); + Logger.info(`Info: worker ${this.workerIndex} prepare test phase for round ${roundIndex} is completed`); } } /** * Perform the test * @param {TestMessage} testMessage start test message - * message = { - * label : label name, - * numb: total number of simulated txs, - * rateControl: rate controller to use - * trim: trim options - * workload: the workload object from the config, - * config: path of the blockchain config file - * totalClients = total number of clients, - * pushUrl = the url for the push gateway - * }; - * @return {Promise} promise object + * @return {Promise} The results of the round execution. */ - async doTest(testMessage) { - Logger.debug('doTest() with:', testMessage.stringify()); + async executeRound(testMessage) { + Logger.debug('Entering executeRound'); - this.beforeTest(testMessage); - - Logger.info('txUpdateTime: ' + this.txUpdateTime); - const self = this; - let txUpdateInter = setInterval( () => { self.txUpdate(); } , self.txUpdateTime); + const workerArguments = testMessage.getWorkerArguments(); + const roundIndex = testMessage.getRoundIndex(); + const roundLabel = testMessage.getRoundLabel(); + Logger.debug(`Worker #${this.workerIndex} starting round #${roundIndex}`); try { + Logger.debug(`Worker #${this.workerIndex} initializing adapter context`); + let context = await this.connector.getContext(testMessage.getRoundLabel(), workerArguments, this.workerIndex); + + // Activate dispatcher + Logger.debug(`Worker #${this.workerIndex} activating TX observer dispatch`); + await this.txObserverDispatch.activate(this.workerIndex, roundIndex, roundLabel); // Configure - let rateController = new RateControl(testMessage.getRateControlSpec(), this.workerIndex, testMessage.getRoundIndex()); - await rateController.init(testMessage.getContent()); + Logger.debug(`Worker #${this.workerIndex} creating rate controller`); + let rateController = new RateControl(testMessage, this.internalTxObserver.getCurrentStatistics(), this.workerIndex); // Run the test loop + Logger.info(`Worker #${this.workerIndex} starting workload loop`); if (testMessage.getRoundDuration()) { const duration = testMessage.getRoundDuration(); // duration in seconds - await this.runDuration(duration, rateController); + await this.runDuration(this.workloadModule, duration, rateController); } else { const number = testMessage.getNumberOfTxs(); - await this.runFixedNumber(number, rateController); + await this.runFixedNumber(this.workloadModule, number, rateController); } + // Deactivate dispatcher + Logger.debug(`Worker #${this.workerIndex} deactivating TX observer dispatch`); + await this.txObserverDispatch.deactivate(); + // Clean up + Logger.debug(`Worker #${this.workerIndex} cleaning up rate controller`); await rateController.end(); + + Logger.debug(`Worker #${this.workerIndex} cleaning up user test module`); await this.workloadModule.cleanupWorkloadModule(); - await this.connector.releaseContext(this.context); - this.clearUpdateInter(txUpdateInter); - - // Return the results and time stamps - if (this.resultStats.length > 0) { - return { - results: this.resultStats[0], - start: this.startTime, - end: this.endTime - }; - } else { - return { - results: TransactionStatistics.createNullDefaultTxStats(), - start: this.startTime, - end: this.endTime - }; - } + + Logger.debug(`Worker #${this.workerIndex} cleaning up adapter context`); + await this.connector.releaseContext(context); + + Logger.debug(`Worker #${this.workerIndex} finished round #${roundIndex}`, this.internalTxObserver.getCurrentStatistics().getCumulativeTxStatistics()); + return this.internalTxObserver.getCurrentStatistics(); } catch (err) { - this.clearUpdateInter(txUpdateInter); - Logger.info(`Worker [${this.workerIndex}] encountered an error: ${(err.stack ? err.stack : err)}`); + Logger.error(`Unexpected error in worker #${this.workerIndex}: ${(err.stack || err)}`); throw err; - } finally { - this.txReset(); } } } diff --git a/packages/caliper-core/lib/worker/rate-control/compositeRate.js b/packages/caliper-core/lib/worker/rate-control/compositeRate.js index 2d38e13900..93231e2fbd 100644 --- a/packages/caliper-core/lib/worker/rate-control/compositeRate.js +++ b/packages/caliper-core/lib/worker/rate-control/compositeRate.js @@ -15,101 +15,84 @@ 'use strict'; const RateInterface = require('./rateInterface.js'); const RateControl = require('./rateControl.js'); -const logger = require('../../common/utils/caliper-utils').getLogger('compositeRate.js'); +const TransactionStatisticsCollector = require('../../common/core/transaction-statistics-collector'); +const logger = require('../../common/utils/caliper-utils').getLogger('composite-rate-controller'); /** * Encapsulates a controller and its scheduling information. * - * Time related values are expressed in millisecond! - * - * @property {number} weight The weight associated with the controller. - * @property {boolean} last Indicates whether the controller is the last in the round. - * @property {object} controllerOptions The supplied options for the controller. + * @property {boolean} isLast Indicates whether the controller is the last in the round. * @property {RateControl} controller The controller instance. - * @property {number} firstTxIndex The first Tx index associated with the controller. - * It is used to calculate the adjusted Tx index passed to the controller. - * @property {number} startTimeDifference The difference between the start time of the round and the controller. - * It is used to calculate the adjusted start time passed to the controller. - * @property {number} lastTxIndex The last Tx index associated with the controller based on its weight. - * Only used in Tx number-based rounds. - * @property {number} relFinishTime The finish time of the controller based on its weight, relative to the start time of the round. - * Only used in duration-based rounds. + * @property {number} lastTxIndex The last TX index associated with the controller based on its weight. Only used in Tx number-based rounds. + * @property {number} relFinishTime The finish time of the controller based on its weight, relative to the start time of the round. Only used in duration-based rounds. + * @property {TransactionStatisticsCollector} txStatSubCollector The TX stat (sub-)collector associated with the sub-controller. */ class ControllerData { /** * Initialize a new instance of the ControllerData class. - * @param {number} weight The weight associated with the controller. - * @param {object} controllerOptions The specified options for the controller. - * @param {RateControl} controller The controller object. + * @param {RateControl} controller The controller instance. + * @param {TransactionStatisticsCollector} txStatSubCollector The TX stat (sub-)collector associated with the sub-controller. */ - constructor(weight, controllerOptions, controller) { - this.weight = weight; + constructor(controller, txStatSubCollector) { this.isLast = false; - this.controllerOptions = controllerOptions; this.controller = controller; - this.firstTxIndex = 0; // correct default value for the first sub-controller - this.startTimeDifference = 0; // correct default value for the first sub-controller this.lastTxIndex = 0; this.relFinishTime = 0; + this.txStatSubCollector = txStatSubCollector; } } /** * Composite rate controller for applying different rate controllers after one an other in the same round. * - * Time related values are expressed in millisecond! - * - * @property {object} options The user-supplied options for the controller. - * @property {number[]} options.weights The list of weights for the different controllers. - * @property {object[]} options.rateControllers The list of descriptors of the controllers. - * @property {boolean} options.logChange Indicates whether to log when switching to a new controller. - * @property {ControllerData[]} controllers The list of relevant controllers and their scheduling information. + * @property {ControllerData[]} controllers The collection of relevant controllers and their scheduling information. * @property {number} activeControllerIndex The index of the currently active controller. - * @property {number} clientIdx The index of the current client. * @property {function} controllerSwitch Duration-based or Tx number-based function for handling controller switches. + * + * @extends RateInterface */ class CompositeRateController extends RateInterface{ /** - * Creates a new instance of the CompositeRateController class. - * @constructor - * @param {object} opts Options for the rate controller. - * @param {number} clientIdx The 0-based index of the client who instantiates the controller. - * @param {number} roundIdx The 1-based index of the round the controller is instantiated in. - * @throws {Error} Throws error if there is a problem with the weight and/or controller options. + * Initializes the rate controller instance. + * @param {TestMessage} testMessage start test message + * @param {TransactionStatisticsCollector} stats The TX stats collector instance. + * @param {number} workerIndex The 0-based index of the worker node. */ - constructor(opts, clientIdx, roundIdx) { - super(opts); + constructor(testMessage, stats, workerIndex) { + super(testMessage, stats, workerIndex); this.controllers = []; this.activeControllerIndex = 0; - this.clientIdx = clientIdx + 1; - this.roundIndex = roundIdx; this.controllerSwitch = null; - this.logControllerChange = (this.options.logChange && - typeof(this.options.logChange) === 'boolean' && this.options.logChange) || false; - this.__prepareControllers(); + this._prepareControllers(); } /** - * Internal method for preparing the controllers for further use. + * Internal method for preparing the sub-controllers and their scheduling information. * @private */ - __prepareControllers() { + _prepareControllers() { let weights = this.options.weights; let rateControllers = this.options.rateControllers; if (!Array.isArray(weights) || !Array.isArray(rateControllers)) { - throw new Error('Weight and controller definitions must be arrays.'); + let msg = 'Weight and controller definitions must be arrays.'; + logger.error(msg); + throw new Error(msg); } if (weights.length !== rateControllers.length) { - throw new Error('The number of weights and controllers must be the same.'); + let msg = 'The number of weights and controllers must be the same.'; + logger.error(msg); + throw new Error(msg); } const nan = weights.find(w => isNaN(Number(w))); if (nan) { - throw new Error('Not-a-number element among weights: ' + nan); + let msg = `Not-a-number element among weights: ${nan}`; + logger.error(msg); + throw new Error(msg); } // store them explicitly as numbers to avoid surprises later @@ -119,152 +102,140 @@ class CompositeRateController extends RateInterface{ // zero weights should be fine to allow easy temporary removal of controllers from the config file const negativeWeight = weights.find(w => w < 0); if (negativeWeight) { - throw new Error('Negative element among weights: ' + negativeWeight); + let msg = `Negative element among weights: ${negativeWeight}`; + logger.error(msg); + throw new Error(msg); } // normalize weights const weightSum = weights.reduce((prev, w) => prev + w, 0); if (weightSum === 0) { - throw new Error('Every weight is zero.'); + let msg = 'Every weight is zero.'; + logger.error(msg); + throw new Error(msg); } weights = weights.map(w => w / weightSum); + // pre-set switch logic to avoid an other condition check during rate control + this.controllerSwitch = this.roundConfig.txNumber ? this._controllerSwitchForTxNumber : this._controllerSwitchForDuration; + + let currentWeightSum = 0; // create controller instances, skip zero-weight cases for (let i = 0; i < weights.length; ++i) { if (weights[i] === 0) { continue; } - let info = new ControllerData(weights[i], rateControllers[i], new RateControl(rateControllers[i], this.clientIdx, this.roundIndex)); - this.controllers.push(info); + let currentWeight = weights[i]; + let currentControllerOptions = rateControllers[i]; + + currentWeightSum += currentWeight; + + // create a modified copy of the round configuration according to the weights + let controllerRoundConfig = Object.assign({}, this.roundConfig); + + // lie to the sub-controller that it's the only one + controllerRoundConfig.rateControl = currentControllerOptions; + + // create the sub-collector to be activated later, and add it to the current stats collector (which could be another sub-collector) + let statSubCollector = new TransactionStatisticsCollector(this.workerIndex, this.roundIndex); + this.stats.addSubCollector(statSubCollector); + + // scale down number of TXs or the duration + if (this.roundConfig.txNumber) { + controllerRoundConfig.txNumber = Math.floor(this.roundConfig.txNumber * currentWeight); + + // the sub-controller is initialized with the TX stat sub-collector, which is inactive at this point (i.e., the round hasn't started for it) + let subcontroller = new RateControl(rateControllers[i], statSubCollector, this.workerIndex, this.roundIndex, this.numberOfWorkers, controllerRoundConfig); + let controllerData = new ControllerData(subcontroller, statSubCollector); + + // the sub-controller should be switched after this TX index + controllerData.lastTxIndex = Math.floor(this.roundConfig.txNumber * currentWeightSum); + this.controllers.push(controllerData); + } else { + controllerRoundConfig.txDuration = Math.floor(this.roundConfig.txDuration * currentWeight); + + // the sub-controller is initialized with the TX stat sub-collector, which is inactive at this point (i.e., the round hasn't started for it) + let subcontroller = new RateControl(rateControllers[i], statSubCollector, this.workerIndex, this.roundIndex, this.numberOfWorkers, controllerRoundConfig); + let controllerData = new ControllerData(subcontroller, statSubCollector); + + // the sub-controller should be switched "around" this time + controllerData.relFinishTime = Math.floor(this.roundConfig.txDuration * 1000 * currentWeightSum); + this.controllers.push(controllerData); + } } // mark the last controller this.controllers[this.controllers.length - 1].isLast = true; + + // activate the TX stats sub-collector of the first sub-controller, i.e., start the round for it (and the TX event processing) + this.controllers[0].txStatSubCollector.activate(); } /** * Internal method for switching controller (when needed) in a duration-based round. - * @param {number} start The start time of the round provided by the client module. - * @param {number} idx The index of the current Tx provided by the client module. * @private * @async */ - async __controllerSwitchForDuration(start, idx) { + async _controllerSwitchForDuration() { + const roundStartTime = this.stats.getRoundStartTime(); + const currentIndex = this.stats.getTotalSubmittedTx(); let active = this.controllers[this.activeControllerIndex]; + const timeNow = Date.now(); - if (active.isLast || ((timeNow - start) < active.relFinishTime)) { + // don't switch from the last controller, or if it isn't time yet + if (active.isLast || ((timeNow - roundStartTime) < active.relFinishTime)) { return; } + // clean up previous sub-controller and its TX stat collector + active.txStatSubCollector.deactivate(); await active.controller.end(); + // activate next sub-controller and its TX stat collector this.activeControllerIndex++; active = this.controllers[this.activeControllerIndex]; - active.firstTxIndex = idx; - active.startTimeDifference = Date.now() - start; - if (this.logControllerChange) { - logger.debug(`Switching controller in Client#${this.clientIdx} at Tx#${idx} after ${active.startTimeDifference}ms.`); - } + active.txStatSubCollector.activate(); + + logger.debug(`Switching controller in worker #${this.workerIndex} for round #${this.roundIndex} at TX #${currentIndex} after ${Date.now() - roundStartTime} ms.`); } /** - * Internal method for switching controller (when needed) in a Tx number-based round. - * @param {number} start The start time of the round provided by the client module. - * @param {number} idx The index of the current Tx provided by the client module. + * Internal method for switching controller (when needed) in a TX number-based round. * @private * @async */ - async __controllerSwitchForTxNumber(start, idx) { + async _controllerSwitchForTxNumber() { + const roundStartTime = this.stats.getRoundStartTime(); + const currentIndex = this.stats.getTotalSubmittedTx(); let active = this.controllers[this.activeControllerIndex]; - if (active.isLast || (idx <= active.lastTxIndex)) { + + // don't switch from the last controller, or if it isn't "time" yet + if (active.isLast || (currentIndex <= active.lastTxIndex)) { return; } + // clean up previous sub-controller and its TX stat collector + active.txStatSubCollector.deactivate(); await active.controller.end(); + // activate next sub-controller and its TX stat collector this.activeControllerIndex++; active = this.controllers[this.activeControllerIndex]; - active.firstTxIndex = idx ; - active.startTimeDifference = Date.now() - start; - if (this.logControllerChange) { - logger.debug(`Switching controller in Client#${this.clientIdx} at Tx#${idx} after ${active.startTimeDifference}ms.`); - } - } - - /** - * Initializes the rate controller. - * - * @param {object} msg Client options with adjusted per-client load settings. - * @param {string} msg.type The type of the message. Currently always 'test' - * @param {string} msg.label The label of the round. - * @param {object} msg.rateControl The rate control to use for the round. - * @param {number} msg.trim The number/seconds of transactions to trim from the results. - * @param {object} msg.args The user supplied arguments for the round. - * @param {string} msg.cb The path of the user's callback module. - * @param {string} msg.config The path of the network's configuration file. - * @param {number} msg.numb The number of transactions to generate during the round. - * @param {number} msg.txDuration The length of the round in SECONDS. - * @param {number} msg.totalClients The number of clients executing the round. - * @param {number} msg.clients The number of clients executing the round. - * @param {object} msg.clientArgs Arguments for the client. - * @param {number} msg.clientIdx The 0-based index of the current client. - * @param {number} msg.roundIdx The 1-based index of the current round. - * @async - */ - async init(msg) { - let currentSum = 0; - this.clientIdx = msg.clientIdx + 1; - - // pre-set switch logic to avoid an other condition check during rate control - this.controllerSwitch = msg.numb ? this.__controllerSwitchForTxNumber : this.__controllerSwitchForDuration; - - for (let i = 0; i < this.controllers.length; ++i) { - let controllerData = this.controllers[i]; - currentSum += controllerData.weight; + active.txStatSubCollector.activate(); - // create a modified copy of the client options according to the weights - let controllerMsg = Object.assign({}, msg); - controllerMsg.rateControl = controllerData.controllerOptions; - - // scale down number of txs or duration - if (msg.numb) { - controllerMsg.numb = Math.floor(msg.numb * controllerData.weight); - controllerData.lastTxIndex = Math.floor(msg.numb * currentSum); - } else { - controllerMsg.txDuration = Math.floor(msg.txDuration * controllerData.weight); - controllerData.relFinishTime = Math.floor(msg.txDuration * 1000 * currentSum); - } - - await controllerData.controller.init(controllerMsg); - } - - this.activeControllerIndex = 0; + logger.debug(`Switching controller in worker #${this.workerIndex} for round #${this.roundIndex} at TX #${currentIndex} after ${Date.now() - roundStartTime} ms.`); } /** - * Perform the rate control by delegating to the currently active controller - * and switching controller (if necessary). - * @param {number} start The epoch time at the start of the round (ms precision). - * @param {number} idx Sequence number of the current transaction. - * @param {object[]} recentResults The list of results of recent transactions. - * @param {object[]} resultStats The aggregated stats of previous results. + * Perform the rate control action by blocking the execution for a certain amount of time. + * Delegates to the currently active sub-controller. * @async */ - async applyRateControl(start, idx, recentResults, resultStats) { - await this.controllerSwitch(start, idx); + async applyRateControl() { + await this.controllerSwitch(); const active = this.controllers[this.activeControllerIndex]; - // NOTE: since we don't know much about the transaction indices corresponding to - // the recent results (the list is emptied periodically), pass it as it is - - // if (idx - this.firstTxIndex + 1) >= recentResults.length ==> everything is transparent, the rate controller - // has been running long enough, so every recent result belongs to it - // otherwise ==> some results MUST belong to the previous controller, but we don't know which result index - // corresponds to active.firstTxIndex, maybe none of them, because this phase hasn't produced results yet - - // lie to the controller about the parameters to make this controller transparent - await active.controller.applyRateControl(start + active.startTimeDifference, idx - active.firstTxIndex, - recentResults, resultStats); + await active.controller.applyRateControl(); } /** @@ -272,19 +243,25 @@ class CompositeRateController extends RateInterface{ * @async */ async end() { + // deactivate the TX stats collector of the last controller + this.controllers[this.activeControllerIndex].txStatSubCollector.deactivate(); await this.controllers[this.activeControllerIndex].controller.end(); } } /** - * Creates a new rate controller instance. - * @param {object} opts The rate controller options. - * @param {number} clientIdx The 0-based index of the client who instantiates the controller. - * @param {number} roundIdx The 1-based index of the round the controller is instantiated in. - * @return {RateInterface} The rate controller instance. + * Factory for creating a new rate controller instance. + * @param {TestMessage} testMessage start test message + * @param {TransactionStatisticsCollector} stats The TX stats collector instance. + * @param {number} workerIndex The 0-based index of the worker node. + * @param {number} roundIndex The 0-based index of the current round. + * @param {number} numberOfWorkers The total number of worker nodes. + * @param {object} roundConfig The round configuration object. + * + * @return {RateInterface} The new rate controller instance. */ -function createRateController(opts, clientIdx, roundIdx) { - return new CompositeRateController(opts, clientIdx, roundIdx); +function createRateController(testMessage, stats, workerIndex) { + return new CompositeRateController(testMessage, stats, workerIndex); } module.exports.createRateController = createRateController; diff --git a/packages/caliper-core/lib/worker/rate-control/fixedBacklog.js b/packages/caliper-core/lib/worker/rate-control/fixedBacklog.js deleted file mode 100644 index 164a3dca33..0000000000 --- a/packages/caliper-core/lib/worker/rate-control/fixedBacklog.js +++ /dev/null @@ -1,136 +0,0 @@ -/* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -'use strict'; - -const RateInterface = require('./rateInterface.js'); -const Sleep = require('../../common/utils/caliper-utils').sleep; -const Logger = require('../../common/utils/caliper-utils').getLogger('fixedBacklog.js'); - -/** - * Rate controller for driving at a target loading (backlog transactions). This controller will aim to maintain a defined backlog - * of unfinished transactions by modifying the driven TPS. - */ -class FixedBacklog extends RateInterface { - /** - * Creates a new instance of the FixedBacklog class. - * @constructor - * @param {object} opts Options for the rate controller. - */ - constructor(opts) { - super(opts); - } - - /** - * Initialise the rate controller with a passed msg object - * - Only requires the desired number of unfinished transactions per worker, derived from the total load and number of workers - * @param {object} msg Client options with adjusted per-client load settings. - * @param {string} msg.type The type of the message. Currently always 'test' - * @param {string} msg.label The label of the round. - * @param {object} msg.rateControl The rate control to use for the round. - * @param {number} msg.trim The number/seconds of transactions to trim from the results. - * @param {object} msg.args The user supplied arguments for the round. - * @param {string} msg.cb The path of the user's callback module. - * @param {string} msg.config The path of the network's configuration file. - * @param {number} msg.numb The number of transactions to generate during the round. - * @param {number} msg.txDuration The length of the round in SECONDS. - * @param {number} msg.totalClients The number of clients executing the round. - * @param {number} msg.clients The number of clients executing the round. - * @param {object} msg.clientArgs Arguments for the client. - * @param {number} msg.clientIdx The 0-based index of the current client. - * @param {number} msg.roundIdx The 1-based index of the current round. - * - * @async - */ - async init(msg) { - let tps; - if (this.options.startingTps) { - tps = this.options.startingTps; - } else { - tps = 1; - } - const tpsPerClient = msg.totalClients ? (tps / msg.totalClients) : tps; - this.sleepTime = 1000/tpsPerClient; - const transactionLoad = this.options.transaction_load ? parseInt(this.options.transaction_load) : 10; - this.unfinished_per_worker = msg.totalClients ? (transactionLoad / msg.totalClients) : transactionLoad; - } - - /** - * Perform the rate control action based on knowledge of the start time, current index, and current results.Sleep a suitable time - * @param {number} start, generation time of the first test transaction - * @param {number} idx, sequence number of the current test transaction - * @param {Array} currentResults, current result set not yet reset by txUpdate() callback - * @param {Array} resultStats, result status set formed in txUpdate() callback - * @async - */ - async applyRateControl(start, idx, currentResults, resultStats) { - - // Waiting until successful transactions occur. - if (resultStats.length < 2 || !resultStats[0].succ || !resultStats[0].delay) { - await Sleep(this.sleepTime); - return; - } - - // Get transaction details - const completeTransactions = resultStats[0].length + currentResults.length; // work from all processed results: resultStats[0]=all processed result stats - let unfinished = idx - completeTransactions; - - const targetTransactionLoad = unfinished - this.unfinished_per_worker; - - // Shortcut if we are below the target threshold and need to increase the loading - if (targetTransactionLoad < 0) { - Logger.debug('Difference between current and desired transaction loading: ' + targetTransactionLoad); - return; - } - - // Determine the current TPS - // - use most recent statistics (resultStats[0] is average over complete round, resultStats[1] is the last txUpdate interval) - const resultStatistics = resultStats[1]; - let tps = 0; - if (resultStatistics.hasOwnProperty('final') && resultStatistics.final.hasOwnProperty('last') && resultStatistics.hasOwnProperty('create') && resultStatistics.create.hasOwnProperty('min')) { - tps = (resultStatistics.succ + resultStatistics.fail ) / (resultStatistics.final.last - resultStatistics.create.min); - } - - // Determine the required sleep to reduce the backlog ( deltaTXN * 1/TPS = sleep in seconds to build the desired txn load) - let sleepTime = 0; - if (tps !== 0) { - sleepTime = targetTransactionLoad * 1000 / tps; - } else { - sleepTime = targetTransactionLoad * this.sleepTime; - } - - Logger.debug('Difference between current and desired transaction backlog: ' + targetTransactionLoad); - await Sleep(sleepTime); - } - - /** - * Notify the rate controller about the end of the round. - * @async - */ - async end() { } -} - - -/** - * Creates a new rate controller instance. - * @param {object} opts The rate controller options. - * @param {number} clientIdx The 0-based index of the client who instantiates the controller. - * @param {number} roundIdx The 1-based index of the round the controller is instantiated in. - * @return {RateInterface} The rate controller instance. - */ -function createRateController(opts, clientIdx, roundIdx) { - return new FixedBacklog(opts); -} - -module.exports.createRateController = createRateController; diff --git a/packages/caliper-core/lib/worker/rate-control/fixedFeedbackRate.js b/packages/caliper-core/lib/worker/rate-control/fixedFeedbackRate.js index df5264d166..cdc1ac2b7d 100644 --- a/packages/caliper-core/lib/worker/rate-control/fixedFeedbackRate.js +++ b/packages/caliper-core/lib/worker/rate-control/fixedFeedbackRate.js @@ -17,106 +17,89 @@ const RateInterface = require('./rateInterface.js'); const util = require('../../common/utils/caliper-utils'); +// TODO: clarify the implementation logic + /** * This controller will send transactions at a specified fixed interval, * but when too many transactions are unfinished, it will sleep a period * of time. + * + * @property {number} generalSleepTime The time to sleep when in normal operation. + * @property {number} backOffTime Used in the calculation of sleep time when sending TPS needs to be decreased. + * @property {number} unfinishedPerWorker The unfinished TX limit per worker nodes that trigger the decrease of TX sending rate, default 10. + * @property {number} zeroSuccessfulCounter The number of times the controller found in a row that there haven't been any successful TXs yet. + * @property {number} totalSleepTime The total number of milliseconds the controller slept in order to decrease TX sending rate. + * + * @extends RateInterface */ -class FixedFeedbackRateController extends RateInterface{ - /** - * Creates a new instance of the FixedFeedbackRateController class. - * @constructor - * @param {object} opts Options for the rate controller. - */ - constructor(opts) { - super(opts); - } +class FixedFeedbackRateController extends RateInterface { /** - * Initializes the rate controller. - * Only requires the desired TPS from the options. - * - * @param {object} msg Client options with adjusted per-client load settings. - * @param {string} msg.type The type of the message. Currently always 'test' - * @param {string} msg.label The label of the round. - * @param {object} msg.rateControl The rate control to use for the round. - * @param {number} msg.trim The number/seconds of transactions to trim from the results. - * @param {object} msg.args The user supplied arguments for the round. - * @param {string} msg.cb The path of the user's callback module. - * @param {string} msg.config The path of the network's configuration file. - * @param {number} msg.numb The number of transactions to generate during the round. - * @param {number} msg.txDuration The length of the round in SECONDS. - * @param {number} msg.totalClients The number of clients executing the round. - * @param {number} msg.clients The number of clients executing the round. - * @param {object} msg.clientArgs Arguments for the client. - * @param {number} msg.clientIdx The 0-based index of the current client. - * @param {number} msg.roundIdx The 1-based index of the current round. - * - * @async + * Initializes the rate controller instance. + * @param {TestMessage} testMessage start test message + * @param {TransactionStatisticsCollector} stats The TX stats collector instance. + * @param {number} workerIndex The 0-based index of the worker node. */ - async init(msg) { - const tps = this.options.tps; - const tpsPerClient = msg.totalClients ? (tps / msg.totalClients) : tps; - this.sleepTime = (tpsPerClient > 0) ? 1000/tpsPerClient : 0; + constructor(testMessage, stats, workerIndex) { + super(testMessage, stats, workerIndex); - this.sleep_time = this.options.sleep_time ? this.options.sleep_time : 100; + const tps = this.options.tps ? parseInt(this.options.tps) : 10; + const tpsPerClient = tps / this.numberOfWorkers; - const maximumLoad = this.options.maximum_transaction_load ? parseInt(this.options.maximum_transaction_load) : 100; - this.unfinished_per_worker = msg.totalClients ? (maximumLoad / msg.totalClients) : maximumLoad; - this.zero_succ_count = 0; + this.generalSleepTime = (tpsPerClient > 0) ? 1000 / tpsPerClient : 0; + this.backOffTime = this.options.sleep_time || 100; - this.total_sleep_time = 0; + const transactionLoad = this.options.transactionLoad ? this.options.transactionLoad : 10; + this.unfinishedPerWorker = (transactionLoad / this.numberOfWorkers); + this.zeroSuccessfulCounter = 0; + this.totalSleepTime = 0; } /** - * Perform the rate control action based on knowledge of the start time, current index, and current results. Sleeps a suitable time. - * @param {number} start The epoch time at the start of the round (ms precision). - * @param {number} idx Sequence number of the current transaction. - * @param {object[]} recentResults The list of results of recent transactions. - * @param {object[]} resultStats The aggregated stats of previous results. + * Perform the rate control action by blocking the execution for a certain amount of time. * @async - */ - async applyRateControl(start, idx, recentResults, resultStats) { - if(this.sleepTime === 0 || idx < this.unfinished_per_worker) { + */ + async applyRateControl() { + let currentSubmitted = this.stats.getTotalSubmittedTx(); + if (this.generalSleepTime === 0 || currentSubmitted < this.unfinishedPerWorker) { return; } - let diff = (this.sleepTime * idx - ((Date.now() - this.total_sleep_time) - start)); - if( diff > 5) { - await util.sleep(diff); + if (this.stats.getTotalFinishedTx() === 0) { return; } - if(resultStats.length === 0) { + const unfinished = currentSubmitted - this.stats.getTotalFinishedTx(); + if (unfinished < this.unfinishedPerWorker / 2) { return; } - let stats = resultStats[0]; - let unfinished = idx - (stats.succ + stats.fail); - - if(unfinished < this.unfinished_per_worker / 2) { + let diff = (this.generalSleepTime * currentSubmitted - ((Date.now() - this.totalSleepTime) - this.stats.getRoundStartTime())); + if (diff > 5) { + await util.sleep(diff); return; } + // Determines the sleep time for waiting until // successful transactions occur - if(resultStats.length > 1 && resultStats[1].succ === 0) { - this.zero_succ_count++; + if (this.stats.getTotalSuccessfulTx() === 0) { + this.zeroSuccessfulCounter++; for(let i = 30; i > 0; --i) { - if(this.zero_succ_count >= i) { - this.total_sleep_time += i * this.sleep_time; - await util.sleep(i * this.sleep_time); + if(this.zeroSuccessfulCounter >= i) { + this.totalSleepTime += i * this.backOffTime; + await util.sleep(i * this.backOffTime); return; } } } - this.zero_succ_count = 0; + this.zeroSuccessfulCounter = 0; // Determines the sleep time according to the current number of // unfinished transactions with the configure one. - for(let i = 10; i > 0; --i) { - if(unfinished >= i * this.unfinished_per_worker) { - this.total_sleep_time += i * this.sleep_time; - await util.sleep(i * this.sleep_time); + for (let i = 10; i > 0; --i) { + if (unfinished >= i * this.unfinishedPerWorker) { + this.totalSleepTime += i * this.backOffTime; + await util.sleep(i * this.backOffTime); return; } } @@ -130,14 +113,15 @@ class FixedFeedbackRateController extends RateInterface{ } /** - * Creates a new rate controller instance. - * @param {object} opts The rate controller options. - * @param {number} clientIdx The 0-based index of the client who instantiates the controller. - * @param {number} roundIdx The 1-based index of the round the controller is instantiated in. - * @return {RateInterface} The rate controller instance. + * Factory for creating a new rate controller instance. + * @param {TestMessage} testMessage start test message + * @param {TransactionStatisticsCollector} stats The TX stats collector instance. + * @param {number} workerIndex The 0-based index of the worker node. + * + * @return {RateInterface} The new rate controller instance. */ -function createRateController(opts, clientIdx, roundIdx) { - return new FixedFeedbackRateController(opts); +function createRateController(testMessage, stats, workerIndex) { + return new FixedFeedbackRateController(testMessage, stats, workerIndex); } module.exports.createRateController = createRateController; diff --git a/packages/caliper-core/lib/worker/rate-control/fixedLoad.js b/packages/caliper-core/lib/worker/rate-control/fixedLoad.js new file mode 100644 index 0000000000..b90466e4c6 --- /dev/null +++ b/packages/caliper-core/lib/worker/rate-control/fixedLoad.js @@ -0,0 +1,112 @@ +/* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +'use strict'; + +const RateInterface = require('./rateInterface.js'); +const Sleep = require('../../common/utils/caliper-utils').sleep; +const Logger = require('../../common/utils/caliper-utils').getLogger('fixedBacklog.js'); + +/** + * Rate controller for driving at a target load (backlog TXs). This controller will aim to maintain a defined backlog + * of unfinished TXs by modifying the driven TPS. + * + * @property {number} startTps The start TPS driving rate, default 5. + * @property {number} transactionLoad The unfinished TX limit per worker nodes that trigger the decrease of TX sending rate, default 10. + * + * @extends RateInterface + */ +class FixedLoad extends RateInterface { + + /** + * Initializes the rate controller instance. + * @param {TestMessage} testMessage start test message + * @param {TransactionStatisticsCollector} stats The TX stats collector instance. + * @param {number} workerIndex The 0-based index of the worker node. + */ + constructor(testMessage, stats, workerIndex) { + super(testMessage, stats, workerIndex); + + const tps = this.options.startTps ? parseInt(this.options.startTps) : 5; + const tpsPerClient = tps / this.numberOfWorkers; + this.sleepTime = 1000 / tpsPerClient; + const transactionLoad = this.options.transactionLoad ? parseInt(this.options.transactionLoad) : 10; + this.targetLoad = transactionLoad / this.numberOfWorkers; + } + + /** + * Perform the rate control action + * @async + */ + async applyRateControl() { + + // Waiting until transactions have completed. + if (this.stats.getTotalFinishedTx() === 0) { + await Sleep(this.sleepTime); + return; + } + + // Get transaction details + let unfinished = this.stats.getTotalSubmittedTx() - this.stats.getTotalFinishedTx(); + + // Shortcut if we are below the target threshold + if (unfinished < this.unfinishedPerWorker) { + return; + } + + const targetLoadDifference = unfinished - this.targetLoad; + + // Shortcut if we are below the target threshold and need to increase the loading + if (targetLoadDifference < 0) { + Logger.debug('Difference between current and desired transaction loading: ' + targetLoadDifference); + return; + } + + // Determine the current TPS + const completedTransactions = this.stats.getTotalSuccessfulTx() + this.stats.getTotalFailedTx(); + const latency = (this.stats.getTotalLatencyForSuccessful() + this.stats.getTotalLatencyForFailed()) / 1000; + const tps = (completedTransactions / latency); + + // Determine the required sleep to reduce the backlog ( deltaTXN * 1/TPS = sleep in seconds to build the desired txn load) + let sleepTime = 0; + if (tps !== 0) { + sleepTime = targetLoadDifference * 1000 / tps; + } else { + sleepTime = targetLoadDifference * this.sleepTime; + } + + Logger.debug('Difference between current and desired transaction backlog: ' + targetLoadDifference); + await Sleep(sleepTime); + } + + /** + * Notify the rate controller about the end of the round. + * @async + */ + async end() { } +} + +/** + * Factory for creating a new rate controller instance. + * @param {TestMessage} testMessage start test message + * @param {TransactionStatisticsCollector} stats The TX stats collector instance. + * @param {number} workerIndex The 0-based index of the worker node. + * + * @return {RateInterface} The new rate controller instance. + */ +function createRateController(testMessage, stats, workerIndex) { + return new FixedLoad(testMessage, stats, workerIndex); +} + +module.exports.createRateController = createRateController; diff --git a/packages/caliper-core/lib/worker/rate-control/fixedRate.js b/packages/caliper-core/lib/worker/rate-control/fixedRate.js index ec169c3c00..174faef170 100644 --- a/packages/caliper-core/lib/worker/rate-control/fixedRate.js +++ b/packages/caliper-core/lib/worker/rate-control/fixedRate.js @@ -18,71 +18,41 @@ let RateInterface = require('./rateInterface.js'); let Sleep = require('../../common/utils/caliper-utils').sleep; /** - * This controller will send transactions at a specified fixed interval. + * Rate controller that sends transactions at a fixed rate. * - * The TPS rate must be specified within the options for the controller type: - * "rateControl" : [{"type": "fixed-rate", "opts": {"tps" : 10}}] + * @property {number} sleepTime The number of milliseconds to sleep to keep the specified TX sending rate. + * + * @extends RateInterface * */ class FixedRate extends RateInterface { /** - * Creates a new instance of the FixedRate class. - * @constructor - * @param {object} opts Options for the rate controller. + * Initializes the rate controller instance. + * @param {TestMessage} testMessage start test message + * @param {TransactionStatisticsCollector} stats The TX stats collector instance. + * @param {number} workerIndex The 0-based index of the worker node. */ - constructor(opts) { - super(opts); - } + constructor(testMessage, stats, workerIndex) { + super(testMessage, stats, workerIndex); - /** - * Initializes the rate controller. - * - * @param {object} msg Client options with adjusted per-client load settings. - * @param {string} msg.type The type of the message. Currently always 'test' - * @param {string} msg.label The label of the round. - * @param {object} msg.rateControl The rate control to use for the round. - * @param {number} msg.trim The number/seconds of transactions to trim from the results. - * @param {object} msg.args The user supplied arguments for the round. - * @param {string} msg.cb The path of the user's callback module. - * @param {string} msg.config The path of the network's configuration file. - * @param {number} msg.numb The number of transactions to generate during the round. - * @param {number} msg.txDuration The length of the round in SECONDS. - * @param {number} msg.totalClients The number of clients executing the round. - * @param {number} msg.clients The number of clients executing the round. - * @param {object} msg.clientArgs Arguments for the client. - * @param {number} msg.clientIdx The 0-based index of the current client. - * @param {number} msg.roundIdx The 1-based index of the current round. - * @async - */ - async init(msg) { - // Use the passed tps option - const tps = this.options.tps; - const tpsPerClient = msg.totalClients ? (tps / msg.totalClients) : tps; + const tps = this.options.tps ? this.options.tps : 10; + const tpsPerClient = tps / this.numberOfWorkers; this.sleepTime = (tpsPerClient > 0) ? 1000/tpsPerClient : 0; } /** - * Perform the rate control action based on knowledge of the start time, current index, and current results. - * - Sleep a suitable time according to the required transaction generation time - * @param {number} start The epoch time at the start of the round (ms precision). - * @param {number} idx Sequence number of the current transaction. - * @param {object[]} recentResults The list of results of recent transactions. - * @param {object[]} resultStats The aggregated stats of previous results. - * @async - */ - async applyRateControl(start, idx, recentResults, resultStats) { - if(this.sleepTime === 0) { - return; - } - let diff = (this.sleepTime * idx - (Date.now() - start)); - if(diff<=5 && idx % 100 === 0) { - await Sleep(5); + * Perform the rate control action by blocking the execution for a certain amount of time. + * @async + */ + async applyRateControl() { + if (this.sleepTime === 0) { return; } - if( diff > 5) { - await Sleep(diff); - } + + const totalSubmitted = this.stats.getTotalSubmittedTx(); + const diff = (this.sleepTime * totalSubmitted - (Date.now() - this.stats.getRoundStartTime())); + await Sleep(diff); } /** @@ -93,14 +63,15 @@ class FixedRate extends RateInterface { } /** - * Creates a new rate controller instance. - * @param {object} opts The rate controller options. - * @param {number} clientIdx The 0-based index of the client who instantiates the controller. - * @param {number} roundIdx The 1-based index of the round the controller is instantiated in. - * @return {RateInterface} The rate controller instance. + * Factory for creating a new rate controller instance. + * @param {TestMessage} testMessage start test message + * @param {TransactionStatisticsCollector} stats The TX stats collector instance. + * @param {number} workerIndex The 0-based index of the worker node. + * + * @return {RateInterface} The new rate controller instance. */ -function createRateController(opts, clientIdx, roundIdx) { - return new FixedRate(opts); +function createRateController(testMessage, stats, workerIndex) { + return new FixedRate(testMessage, stats, workerIndex); } module.exports.createRateController = createRateController; diff --git a/packages/caliper-core/lib/worker/rate-control/linearRate.js b/packages/caliper-core/lib/worker/rate-control/linearRate.js index f309fe3e42..29fca9d49b 100644 --- a/packages/caliper-core/lib/worker/rate-control/linearRate.js +++ b/packages/caliper-core/lib/worker/rate-control/linearRate.js @@ -15,94 +15,68 @@ 'use strict'; const RateInterface = require('./rateInterface.js'); -const util = require('../../common/utils/caliper-utils'); +let Sleep = require('../../common/utils/caliper-utils').sleep; /** * Rate controller for generating a linearly changing workload. * - * @property {object} options The user-supplied options for the controller. * @property {number} startingSleepTime The sleep time for the first transaction in milliseconds. * @property {number} gradient The gradient of the line. + * @property {function} _interpolate Function for calculating the current time to sleep (either from TX index, or from the elapsed time). + * + * @extends RateInterface */ -class LinearRateController extends RateInterface{ +class LinearRateController extends RateInterface { /** - * Creates a new instance of the LinearRateController class. - * @constructor - * @param {object} opts Options for the rate controller. + * Initializes the rate controller instance. + * @param {TestMessage} testMessage The testMessage passed for the round execution + * @param {TransactionStatisticsCollector} stats The TX stats collector instance. + * @param {number} workerIndex The 0-based index of the worker node. + * @param {number} roundIndex The 0-based index of the current round. + * @param {number} numberOfWorkers The total number of worker nodes. + * @param {object} roundConfig The round configuration object. */ - constructor(opts) { - super(opts); + constructor(testMessage, stats, workerIndex) { + super(testMessage, stats, workerIndex); + + // distributing TPS among clients + let startingTps = Number(this.options.startingTps) / this.numberOfWorkers; + let finishingTps = Number(this.options.finishingTps) / this.numberOfWorkers; + this.startingSleepTime = 1000 / startingTps; + let finishingSleepTime = 1000 / finishingTps; + + // based on linear interpolation between two points with (time/index, sleep time) axes + let duration = this.testMessage.getNumberOfTxs() || (this.testMessage.getRoundDuration() * 1000); + this.gradient = (finishingSleepTime - this.startingSleepTime) / duration; + + // to avoid constant if/else check with the same result + this._interpolate = this.testMessage.getNumberOfTxs() ? this._interpolateFromIndex : this._interpolateFromTime; } /** * Interpolates the current sleep time from the transaction index. - * @param {number} start The epoch time at the start of the round (ms precision). - * @param {number} idx Sequence number of the current transaction. * @return {number} The interpolated sleep time. - * @private */ - _interpolateFromIndex(start, idx) { - return this.startingSleepTime + idx * this.gradient; + _interpolateFromIndex() { + return this.startingSleepTime + this.stats.getTotalSubmittedTx() * this.gradient; } /** * Interpolates the current sleep time from the elapsed time. - * @param {number} start The epoch time at the start of the round (ms precision). - * @param {number} idx Sequence number of the current transaction. * @return {number} The interpolated sleep time. - * @private - */ - _interpolateFromTime(start, idx) { - return this.startingSleepTime + (Date.now() - start) * this.gradient; - } - - /** - * Initializes the rate controller. - * - * @param {object} msg Client options with adjusted per-client load settings. - * @param {string} msg.type The type of the message. Currently always 'test' - * @param {string} msg.label The label of the round. - * @param {object} msg.rateControl The rate control to use for the round. - * @param {number} msg.trim The number/seconds of transactions to trim from the results. - * @param {object} msg.args The user supplied arguments for the round. - * @param {string} msg.cb The path of the user's callback module. - * @param {string} msg.config The path of the network's configuration file. - * @param {number} msg.numb The number of transactions to generate during the round. - * @param {number} msg.txDuration The length of the round in SECONDS. - * @param {number} msg.totalClients The number of clients executing the round. - * @param {number} msg.clients The number of clients executing the round. - * @param {object} msg.clientArgs Arguments for the client. - * @param {number} msg.clientIdx The 0-based index of the current client. - * @param {number} msg.roundIdx The 1-based index of the current round. - * @async */ - async init(msg) { - // distributing TPS among clients - let startingTps = Number(this.options.startingTps) / msg.totalClients; - let finishingTps = Number(this.options.finishingTps) / msg.totalClients; - this.startingSleepTime = 1000 / startingTps; - let finishingSleepTime = 1000 / finishingTps; - - // based on linear interpolation between two points with (time/index, sleep time) axes - let duration = msg.numb? msg.numb : msg.txDuration * 1000; - this.gradient = (finishingSleepTime - this.startingSleepTime) / duration; - - // to avoid constant if/else check with the same result - this._interpolate = msg.numb ? this._interpolateFromIndex : this._interpolateFromTime; + _interpolateFromTime() { + return this.startingSleepTime + (Date.now() - this.stats.getRoundStartTime()) * this.gradient; } /** - * Perform the rate control. - * @param {number} start The epoch time at the start of the round (ms precision). - * @param {number} idx Sequence number of the current transaction. - * @param {object[]} recentResults The list of results of recent transactions. - * @param {object[]} resultStats The aggregated stats of previous results. + * Perform the rate control action by blocking the execution for a certain amount of time. * @async */ - async applyRateControl(start, idx, recentResults, resultStats) { - let currentSleepTime = this._interpolate(start, idx); + async applyRateControl() { + let currentSleepTime = this._interpolate(); if (currentSleepTime > 5) { - await util.sleep(currentSleepTime); + await Sleep(currentSleepTime); } } @@ -114,14 +88,15 @@ class LinearRateController extends RateInterface{ } /** - * Creates a new rate controller instance. - * @param {object} opts The rate controller options. - * @param {number} clientIdx The 0-based index of the client who instantiates the controller. - * @param {number} roundIdx The 1-based index of the round the controller is instantiated in. - * @return {RateInterface} The rate controller instance. + * Factory for creating a new rate controller instance. + * @param {TestMessage} testMessage start test message + * @param {TransactionStatisticsCollector} stats The TX stats collector instance. + * @param {number} workerIndex The 0-based index of the worker node. + * + * @return {RateInterface} The new rate controller instance. */ -function createRateController(opts, clientIdx, roundIdx) { - return new LinearRateController(opts); +function createRateController(testMessage, stats, workerIndex) { + return new LinearRateController(testMessage, stats, workerIndex); } module.exports.createRateController = createRateController; diff --git a/packages/caliper-core/lib/worker/rate-control/maxRate.js b/packages/caliper-core/lib/worker/rate-control/maxRate.js index 6dcc5ae338..bd9375c7bf 100644 --- a/packages/caliper-core/lib/worker/rate-control/maxRate.js +++ b/packages/caliper-core/lib/worker/rate-control/maxRate.js @@ -19,176 +19,149 @@ const Sleep = require('../../common/utils/caliper-utils').sleep; const Logger = require('../../common/utils/caliper-utils').getLogger('maxRate.js'); /** - * Rate controller for driving at a maximum TPS. + * Rate controller for driving at a maximum TPS. This controller will aim to steadily increase the driven TPS to the maximum + * sustainable rate + * + * @property {number} sampleInterval The interval over which to assess if the driven TPS should be stepped up/down, in seconds + * @property {number} tps The starting TPS at which to drive the rate controller, default 5 + * @property {number} step The TPS step size used to increase/decrease TPS rate, default 5 + * @property {boolean} includeFailed A flag to indicate if failed transactions should be considered when assessing the rate change, default true + * + * @extends RateInterface */ class MaxRate extends RateInterface { + /** - * Creates a new instance of the MaxRate class. - * @constructor - * @param {object} opts Options for the rate controller. + * Initializes the rate controller instance. + * @param {TestMessage} testMessage start test message + * @param {TransactionStatisticsCollector} stats The TX stats collector instance. + * @param {number} workerIndex The 0-based index of the worker node. */ - constructor(opts) { - super(opts); + constructor(testMessage, stats, workerIndex) { + super(testMessage, stats, workerIndex); + + // Minimum sample interval (default 10s) + this.sampleInterval = this.options.sampleInterval ? parseInt(this.options.sampleInterval) * 1000 : 10000; + + // Include failed transactions in TPS + this.includeFailed = this.options.includeFailed ? this.options.includeFailed : true; + + // Client TPS + const startTps = this.options.tps ? this.options.tps : 5; + const startTpsPerClient = startTps / this.numberOfWorkers; - // Default sleep - this.sleepTime = 100; + // Client TPS Step + const tpsStep = this.options.step ? this.options.step : 5; + this.step = tpsStep / this.numberOfWorkers; - // Map for TPS observations + // Object for TPS observations this.observedTPS = { previous: 0, current: 0 }; - // Map for TPS settings + // Object for TPS settings this.tpsSettings = { - previous: 0, - current: 0 + previous: startTpsPerClient, + current: startTpsPerClient }; - // MPS for observed stats - this.statistics = { - previous: [], - current: [], - sampleStart: 0 + // Object for observed stats + this.internalStats = { + previousCompletedTotal: 0, + currentCompletedTotal: 0, + previousElapsedTime: 0, + currentElapsedTime: 0, + lastUpdate: 0 }; - - // Minimum sample interval (default 10s) - this.sampleInterval = opts.sampleInterval ? opts.sampleInterval : 10; - - // Include failed transactions in TPS - this.includeFailed = opts.includeFailed ? opts.includeFailed : true; } /** - * Initialise the rate controller with a passed msg object - * @param {object} msg Client options with adjusted per-client load settings. - * @param {string} msg.type The type of the message. Currently always 'test' - * @param {string} msg.label The label of the round. - * @param {object} msg.rateControl The rate control to use for the round. - * @param {number} msg.trim The number/seconds of transactions to trim from the results. - * @param {object} msg.args The user supplied arguments for the round. - * @param {string} msg.cb The path of the user's callback module. - * @param {string} msg.config The path of the network's configuration file. - * @param {number} msg.numb The number of transactions to generate during the round. - * @param {number} msg.txDuration The length of the round in SECONDS. - * @param {number} msg.totalClients The number of clients executing the round. - * @param {number} msg.clients The number of clients executing the round. - * @param {object} msg.clientArgs Arguments for the client. - * @param {number} msg.clientIdx The 0-based index of the current client. - * @param {number} msg.roundIdx The 1-based index of the current round. - * - * @async - */ - async init(msg) { - - - // Client TPS - const startTps = this.options.tps ? this.options.tps : 5; - const startTpsPerClient = msg.totalClients ? (startTps / msg.totalClients) : startTps; - // - Store these - this.tpsSettings.previous = startTpsPerClient; - this.tpsSettings.current = startTpsPerClient; - - // Client TPS Step - const tpsStep = this.options.step ? this.options.step : 5; - this.step = msg.totalClients ? (tpsStep / msg.totalClients) : tpsStep; - } - - /** - * Perform the rate control action based on knowledge of the start time, current index, and current results.Sleep a suitable time - * @param {number} start, generation time of the first test transaction (unused) - * @param {number} idx, sequence number of the current test transaction - * @param {Array} currentResults, current result set not yet reset by txUpdate() callback - * @param {Array} resultStats, result status set formed in txUpdate() callback + * Perform the rate control action * @async */ - async applyRateControl(start, idx, currentResults, resultStats) { + async applyRateControl() { - // Waiting until successful transactions occur. - if (resultStats.length < 2 || !resultStats[0].succ || !resultStats[0].create || !resultStats[0].final) { + // Waiting until transactions have completed. + if (this.stats.getTotalFinishedTx() === 0) { await this.applySleepInterval(); return; } else { - // txUpdate intervals are the only places we can detect changes. This is refreshed, and at that point - // minCreate will increase as we will be dealing with more recent submissions - // First entry - if (this.statistics.current.length === 0) { - this.statistics.previous = resultStats[1]; - this.statistics.current = resultStats[1]; - this.statistics.sampleStart = resultStats[1].create.min; + if (this.internalStats.lastUpdate === 0) { + this.internalStats.lastUpdate = Date.now(); + } + + // Have we waited the required sample interval? + if (this.exceededSampleInterval()) { + let currentCompletedTotal; + if (this.includeFailed) { + currentCompletedTotal = this.stats.getTotalSuccessfulTx() + this.stats.getTotalFailedTx(); + } else { + currentCompletedTotal = this.stats.getTotalSuccessfulTx(); + } + + let currentElapsedTime; + if (this.includeFailed) { + currentElapsedTime = this.stats.getTotalLatencyForSuccessful() + this.stats.getTotalLatencyForFailed(); + } else { + currentElapsedTime = this.stats.getTotalLatencyForSuccessful(); + } + + this.internalStats.currentCompletedTotal = currentCompletedTotal; + this.internalStats.currentElapsedTime = currentElapsedTime; + + const achievedTPS = this.retrieveIntervalTPS(); - const achievedTPS = this.retrieveIntervalTPS(resultStats); + // New TPS results + this.observedTPS.previous = this.observedTPS.current; this.observedTPS.current = achievedTPS; - } - // Only modify when result stats has been updated - if (this.updateOccurred(resultStats)) { - - // Have we waited the required sample interval? - if (this.exceededSampleInterval(resultStats)) { - this.statistics.current = resultStats[1]; - this.statistics.sampleStart = resultStats[1].final.last; - const achievedTPS = this.retrieveIntervalTPS(resultStats); - - // New TPS results - this.observedTPS.previous = this.observedTPS.current; - this.observedTPS.current = achievedTPS; - - Logger.debug(`Observed current worker TPS ${this.observedTPS.current}`); - Logger.debug(`Observed previous worker TPS ${this.observedTPS.previous}`); - - // Action based on transaction rate trajectory (+/-) - const dTxn = this.observedTPS.current - this.observedTPS.previous; - this.tpsSettings.previous = this.tpsSettings.current; - if (dTxn > 0) { - // Keep ramping, try for the new max! - this.tpsSettings.current = this.tpsSettings.current + this.step; - Logger.debug(`Increased worker TPS to ${this.tpsSettings.current}`); - } else { - // Too far, back off and try smaller step size. Need to ensure we drain the backlog too. - this.tpsSettings.current = this.tpsSettings.current - this.step; - this.step = this.step > 0.2 ? this.step / 2 : this.step; - Logger.debug(`Decreased worker TPS to ${this.tpsSettings.current} and step size to ${this.step}`); - } + Logger.debug(`Observed current worker TPS ${this.observedTPS.current}`); + Logger.debug(`Observed previous worker TPS ${this.observedTPS.previous}`); + + // Action based on transaction rate trajectory (+/-) + const dTxn = this.observedTPS.current - this.observedTPS.previous; + this.tpsSettings.previous = this.tpsSettings.current; + if (dTxn > 0) { + // Keep ramping, try for the new max! + this.tpsSettings.current = this.tpsSettings.current + this.step; + Logger.debug(`Increased worker TPS to ${this.tpsSettings.current}`); + } else { + // Too far, back off and try smaller step size. Need to ensure we drain the backlog too. + this.tpsSettings.current = this.tpsSettings.current - this.step; + this.step = this.step > 0.2 ? this.step / 2 : this.step; + Logger.debug(`Decreased worker TPS to ${this.tpsSettings.current} and step size to ${this.step}`); } + + // update internal stats + this.internalStats.lastUpdate = Date.now(); + this.internalStats.previousCompletedTotal = currentCompletedTotal; + this.internalStats.previousElapsedTime = currentElapsedTime; } + } // Continue at fixed TPS within this update interval await this.applySleepInterval(); } - /** - * Check if a txUpdate has occurred - * @param {object} resultStats the result statistics - * @returns {boolean} update boolean - */ - updateOccurred(resultStats) { - return this.statistics.current.create.min !== resultStats[1].create.min; - } - /** * Check if required sample time has been reached - * @param {object} resultStats the result statistics * @returns {boolean} boolean flag */ - exceededSampleInterval(resultStats) { - return resultStats[1].final.last - this.statistics.sampleStart >= this.sampleInterval; + exceededSampleInterval() { + return Date.now() - this.internalStats.lastUpdate >= this.sampleInterval; } /** - * TPS from the previous txUpdate interval statistics - * @param {object} resultStats the passed stats object + * TPS from the completed interval statistics * @return {number} the TPS within the interval */ - retrieveIntervalTPS(resultStats) { - const resultStatistics = resultStats[1]; - if (this.includeFailed) { - return (resultStatistics.succ + resultStatistics.fail) / (resultStatistics.final.last - resultStatistics.create.min); - } else { - return resultStatistics.succ / (resultStatistics.final.last - resultStatistics.create.min); - } + retrieveIntervalTPS() { + const intervalCompleted = this.internalStats.currentCompletedTotal - this.internalStats.previousCompletedTotal; + const intervalLatency = (this.internalStats.currentElapsedTime - this.internalStats.previousElapsedTime) / 1000; + return intervalCompleted / intervalLatency; } /** @@ -203,19 +176,21 @@ class MaxRate extends RateInterface { * Notify the rate controller about the end of the round. * @async */ - async end() { } + async end() { + Logger.info(`End worker TPS ${this.tpsSettings.current}`); + } } - /** - * Creates a new rate controller instance. - * @param {object} opts The rate controller options. - * @param {number} clientIdx The 0-based index of the client who instantiates the controller. - * @param {number} roundIdx The 1-based index of the round the controller is instantiated in. - * @return {RateInterface} The rate controller instance. + * Factory for creating a new rate controller instance. + * @param {TestMessage} testMessage start test message + * @param {TransactionStatisticsCollector} stats The TX stats collector instance. + * @param {number} workerIndex The 0-based index of the worker node. + * + * @return {RateInterface} The new rate controller instance. */ -function createRateController(opts, clientIdx, roundIdx) { - return new MaxRate(opts); +function createRateController(testMessage, stats, workerIndex) { + return new MaxRate(testMessage, stats, workerIndex); } module.exports.createRateController = createRateController; diff --git a/packages/caliper-core/lib/worker/rate-control/noRate.js b/packages/caliper-core/lib/worker/rate-control/noRate.js index 6f4720b0df..7893b70547 100644 --- a/packages/caliper-core/lib/worker/rate-control/noRate.js +++ b/packages/caliper-core/lib/worker/rate-control/noRate.js @@ -15,64 +15,38 @@ 'use strict'; const RateInterface = require('./rateInterface.js'); -const Util = require('../../common/utils/caliper-utils'); +const Sleep = require('../../common/utils/caliper-utils').sleep; /** * Rate controller for pausing load generation for a given time. + * Can only be applied for duration-based rounds! Only meaningful as a sub-controller in a composite rate. * - * Can only be applied for duration-based rounds! + * @property {number} sleepTime The length of the round in milliseconds (i.e., the time to sleep). * - * @property {object} options The user-supplied options for the controller. + * @extends RateInterface */ -class NoRateController extends RateInterface{ +class NoRateController extends RateInterface { /** - * Creates a new instance of the {NoRateController} class. - * @constructor - * @param {object} opts Options for the rate controller. + * Initializes the rate controller instance. + * @param {TestMessage} testMessage start test message + * @param {TransactionStatisticsCollector} stats The TX stats collector instance. + * @param {number} workerIndex The 0-based index of the worker node. */ - constructor(opts) { - super(opts); - this.sleepTime = 0; - } - - /** - * Initializes the rate controller. - * - * @param {object} msg Client options with adjusted per-client load settings. - * @param {string} msg.type The type of the message. Currently always 'test' - * @param {string} msg.label The label of the round. - * @param {object} msg.rateControl The rate control to use for the round. - * @param {number} msg.trim The number/seconds of transactions to trim from the results. - * @param {object} msg.args The user supplied arguments for the round. - * @param {string} msg.cb The path of the user's callback module. - * @param {string} msg.config The path of the network's configuration file. - * @param {number} msg.numb The number of transactions to generate during the round. - * @param {number} msg.txDuration The length of the round in SECONDS. - * @param {number} msg.totalClients The number of clients executing the round. - * @param {number} msg.clients The number of clients executing the round. - * @param {object} msg.clientArgs Arguments for the client. - * @param {number} msg.clientIdx The 0-based index of the current client. - * @param {number} msg.roundIdx The 1-based index of the current round. - * @async - */ - async init(msg) { - if (msg.numb) { + constructor(testMessage, stats, workerIndex) { + super(testMessage, stats, workerIndex); + if (testMessage.getNumberOfTxs()) { throw new Error('The no-rate controller can only be applied for duration-based rounds'); } - this.sleepTime = msg.txDuration * 1000; + this.sleepTime = testMessage.getRoundDuration() * 1000; } /** - * Perform the rate control by sleeping through the round. - * @param {number} start The epoch time at the start of the round (ms precision). - * @param {number} idx Sequence number of the current transaction. - * @param {object[]} recentResults The list of results of recent transactions. - * @param {object[]} resultStats The aggregated stats of previous results. + * Perform the rate control action by blocking the execution for a certain amount of time. * @async */ - async applyRateControl(start, idx, recentResults, resultStats) { - await Util.sleep(this.sleepTime); + async applyRateControl() { + await Sleep(this.sleepTime); } /** @@ -83,14 +57,15 @@ class NoRateController extends RateInterface{ } /** - * Creates a new rate controller instance. - * @param {object} opts The rate controller options. - * @param {number} clientIdx The 0-based index of the client who instantiates the controller. - * @param {number} roundIdx The 1-based index of the round the controller is instantiated in. - * @return {RateInterface} The rate controller instance. + * Factory for creating a new rate controller instance. + * @param {TestMessage} testMessage start test message + * @param {TransactionStatisticsCollector} stats The TX stats collector instance. + * @param {number} workerIndex The 0-based index of the worker node. + * + * @return {RateInterface} The new rate controller instance. */ -function createRateController(opts, clientIdx, roundIdx) { - return new NoRateController(opts); +function createRateController(testMessage, stats, workerIndex) { + return new NoRateController(testMessage, stats, workerIndex); } module.exports.createRateController = createRateController; diff --git a/packages/caliper-core/lib/worker/rate-control/rateControl.js b/packages/caliper-core/lib/worker/rate-control/rateControl.js index 24d0064e77..f4ce4d6634 100644 --- a/packages/caliper-core/lib/worker/rate-control/rateControl.js +++ b/packages/caliper-core/lib/worker/rate-control/rateControl.js @@ -14,86 +14,70 @@ 'use strict'; const CaliperUtils = require('../../common/utils/caliper-utils'); +const RateInterface = require('./rateInterface'); const logger = CaliperUtils.getLogger('rateControl.js'); const builtInControllers = new Map([ - ['fixed-rate', './fixedRate.js'], - ['fixed-backlog', './fixedBacklog.js'], ['composite-rate', './compositeRate.js'], + ['fixed-load', './fixedLoad.js'], + ['fixed-feedback-rate', './fixedFeedbackRate.js'], + ['fixed-rate', './fixedRate.js'], + ['linear-rate', './linearRate.js'], + ['maximum-rate', './maxRate.js'], ['zero-rate', './noRate.js'], ['record-rate', './recordRate.js'], - ['replay-rate', './replayRate.js'], - ['linear-rate', './linearRate.js'], - ['fixed-feedback-rate', './fixedFeedbackRate.js'], - ['maximum-rate', './maxRate.js'] + ['replay-rate', './replayRate.js'] ]); -const RateControl = class { +/** + * Proxy class for creating and managing the configured rate controller. + * + * @property {RateInterface} controller The managed rate controller instance. + * + * @extends RateInterface + */ +class RateControl extends RateInterface { /** - * Instantiates the proxy rate controller and creates the configured rate controller behind it. - * @param {{type:string, opts:object}} rateControl The object describing the rate controller to use. - * @param {number} clientIdx The 0-based index of the client who instantiates the controller. - * @param {number} roundIdx The 1-based index of the round the controller is instantiated in. + * Initializes the rate controller proxy. + * @param {TestMessage} testMessage start test message + * @param {TransactionStatisticsCollector} stats The TX stats collector instance. + * @param {number} workerIndex The 0-based index of the worker node. */ - constructor(rateControl, clientIdx, roundIdx) { - logger.debug(`Creating rate controller for client#${clientIdx} for round#${roundIdx}`, rateControl); + constructor(testMessage, stats, workerIndex) { + super(testMessage, stats, workerIndex); + logger.debug(`Creating rate controller of type ${testMessage.getRateControlSpec().type} for worker #${workerIndex} in round #${testMessage.getRoundIndex()}`, testMessage.getRateControlSpec()); // resolve the type to a module path - let modulePath = builtInControllers.has(rateControl.type) - ? builtInControllers.get(rateControl.type) : CaliperUtils.resolvePath(rateControl.type); + const rateControllerType = testMessage.getRateControlSpec().type; + let modulePath = builtInControllers.has(rateControllerType) + ? builtInControllers.get(testMessage.getRateControlSpec().type) : CaliperUtils.resolvePath(rateControllerType); let factoryFunction = require(modulePath).createRateController; if (!factoryFunction) { - throw new Error(`${rateControl.type} does not export the mandatory factory function`); + throw new Error(`${rateControllerType} does not export the mandatory factory function`); } - this.controller = factoryFunction(rateControl.opts, clientIdx, roundIdx); - } - - /** - * Initializes the rate controller for the round. - * - * @param {object} msg Client options with adjusted per-client load settings. - * @param {string} msg.type The type of the message. Currently always 'test' - * @param {string} msg.label The label of the round. - * @param {object} msg.rateControl The rate control to use for the round. - * @param {number} msg.trim The number/seconds of transactions to trim from the results. - * @param {object} msg.args The user supplied arguments for the round. - * @param {string} msg.cb The path of the user's callback module. - * @param {string} msg.config The path of the network's configuration file. - * @param {number} msg.numb The number of transactions to generate during the round. - * @param {number} msg.txDuration The length of the round in SECONDS. - * @param {number} msg.totalClients The number of clients executing the round. - * @param {number} msg.clients The number of clients executing the round. - * @param {object} msg.clientArgs Arguments for the client. - * @param {number} msg.clientIdx The 0-based index of the current client. - * @param {number} msg.roundIdx The 1-based index of the current round. - * @async - */ - async init(msg) { - await this.controller.init(msg); + this.controller = factoryFunction(testMessage, stats, workerIndex); } /** - * Perform the rate control action based on knowledge of the start time, current index, and previous results. - * @param {number} start The epoch time at the start of the round (ms precision). - * @param {number} idx Sequence number of the current transaction. - * @param {object[]} recentResults The list of results of recent transactions. - * @param {Array} resultStats The aggregated stats of previous results. + * Perform the rate control action by blocking the execution for a certain amount of time. + * Delegates to the underlying rate controller. * @async */ - async applyRateControl(start, idx, recentResults, resultStats) { - await this.controller.applyRateControl(start, idx, recentResults, resultStats); + async applyRateControl() { + await this.controller.applyRateControl(); } /** * Notify the rate controller about the end of the round. + * Delegates to the underlying rate controller. * @async */ async end() { await this.controller.end(); } -}; +} module.exports = RateControl; diff --git a/packages/caliper-core/lib/worker/rate-control/rateInterface.js b/packages/caliper-core/lib/worker/rate-control/rateInterface.js index 801cbb6b5c..21e033c9dc 100644 --- a/packages/caliper-core/lib/worker/rate-control/rateInterface.js +++ b/packages/caliper-core/lib/worker/rate-control/rateInterface.js @@ -15,53 +15,32 @@ 'use strict'; /** - * Rate interface for creating rate controllers + * Base class for rate controllers. */ class RateInterface { /** - * Base class constructor. - * @param {object} opts The rate controller options. - * @constructor + * Initializes the rate controller instance. + * @param {TestMessage} testMessage The testMessage passed for the round execution + * @param {TransactionStatisticsCollector} stats The TX stats collector instance. + * @param {number} workerIndex The 0-based index of the worker node. */ - constructor(opts) { - this.options = opts; + constructor(testMessage, stats, workerIndex) { + this.testMessage = testMessage; + this.stats = stats; + this.workerIndex = workerIndex; + this.controller = testMessage.getRateControlSpec(); + this.options = this.controller.opts; + this.roundIndex = testMessage.getRoundIndex(); + this.numberOfWorkers = testMessage.getWorkersNumber(); } /** - * Initializes the rate controller. - * - * @param {object} msg Client options with adjusted per-client load settings. - * @param {string} msg.type The type of the message. Currently always 'test' - * @param {string} msg.label The label of the round. - * @param {object} msg.rateControl The rate control to use for the round. - * @param {number} msg.trim The number/seconds of transactions to trim from the results. - * @param {object} msg.args The user supplied arguments for the round. - * @param {string} msg.cb The path of the user's callback module. - * @param {string} msg.config The path of the network's configuration file. - * @param {number} msg.numb The number of transactions to generate during the round. - * @param {number} msg.txDuration The length of the round in SECONDS. - * @param {number} msg.totalClients The number of clients executing the round. - * @param {number} msg.clients The number of clients executing the round. - * @param {object} msg.clientArgs Arguments for the client. - * @param {number} msg.clientIdx The 0-based index of the current client. - * @param {number} msg.roundIdx The 1-based index of the current round. + * Perform the rate control action by blocking the execution for a certain amount of time. * @async */ - async init(msg) { - throw new Error('Function \'init\' is not implemented for this rate controller'); - } - - /** - * Perform the rate control action based on knowledge of the start time, current index, and previous results. - * @param {number} start The epoch time at the start of the round (ms precision). - * @param {number} idx Sequence number of the current transaction. - * @param {object[]} recentResults The list of results of recent transactions. - * @param {object[]} resultStats The aggregated stats of previous results. - * @async - */ - async applyRateControl(start, idx, recentResults, resultStats) { - throw new Error('Function \'applyRateControl\' is not implemented for this rate controller'); + async applyRateControl() { + throw new Error('Method \'applyRateControl\' is not implemented for this rate controller'); } /** @@ -69,7 +48,7 @@ class RateInterface { * @async */ async end() { - throw new Error('Function \'end\' is not implemented for this rate controller'); + throw new Error('Method \'end\' is not implemented for this rate controller'); } } diff --git a/packages/caliper-core/lib/worker/rate-control/recordRate.js b/packages/caliper-core/lib/worker/rate-control/recordRate.js index 2b824bf50f..7282b248ea 100644 --- a/packages/caliper-core/lib/worker/rate-control/recordRate.js +++ b/packages/caliper-core/lib/worker/rate-control/recordRate.js @@ -18,7 +18,7 @@ const RateInterface = require('./rateInterface.js'); const RateControl = require('./rateControl'); const fs = require('fs'); const util = require('../../common/utils/caliper-utils'); -const logger = util.getLogger('recordRate.js'); +const logger = util.getLogger('record-rate-controller'); const TEXT_FORMAT = 'TEXT'; const BINARY_BE_FORMAT = 'BIN_BE'; @@ -28,52 +28,57 @@ const supportedFormats = [TEXT_FORMAT, BINARY_BE_FORMAT, BINARY_LE_FORMAT]; /** * Decorator rate controller for recording the rate of an other controller. * - * @property {object} options The user-supplied options for the controller. - * @property {object[]} records The record of times for submitted transactions. + * @property {number[]} records The record of relative times for submitted transactions. * @property {RateControl} rateController The rate controller to record. * @property {string} pathTemplate The template path for the file to record to. - * @property {number} roundIdx The index of the current round. - * @property {number} clientIdx The index of the current client. * @property {string} outputFormat Specifies the output format for the recordings. - * @property {boolean} logEnd Indicates whether log when the records are written to file. + * + * @extends RateInterface */ -class RecordRateController extends RateInterface{ +class RecordRateController extends RateInterface { /** - * Creates a new instance of the {RecordRateController} class. - * @constructor - * @param {object} opts Options for the rate controller. - * @param {number} clientIdx The 0-based index of the client who instantiates the controller. - * @param {number} roundIdx The 1-based index of the round the controller is instantiated in. + * Initializes the rate controller instance. + * @param {TestMessage} testMessage start test message + * @param {TransactionStatisticsCollector} stats The TX stats collector instance. + * @param {number} workerIndex The 0-based index of the worker node. */ - constructor(opts, clientIdx, roundIdx) { - super(opts); - this.roundIdx = roundIdx; - this.clientIdx = clientIdx + 1; + constructor(testMessage, stats, workerIndex) { + super(testMessage, stats, workerIndex); + this.records = []; + // if we know the number of transactions beforehand, pre-allocate the array + if (testMessage.getNumberOfTxs()) { + this.records = new Array(testMessage.getNumberOfTxs()); + this.records.fill(0); + } - if (typeof opts.pathTemplate === 'undefined') { + if (typeof this.options.pathTemplate === 'undefined') { throw new Error('The path to save the recording to is undefined'); } - if (typeof opts.rateController === 'undefined') { + if (typeof this.options.rateController === 'undefined') { throw new Error('The rate controller to record is undefined'); } - this.logEnd = Boolean(opts.logEnd || false); - // check for supported output formats - if (typeof opts.outputFormat === 'undefined') { - logger.warn(`Output format is undefined. Defaulting to ${TEXT_FORMAT} format`); + if (typeof this.options.outputFormat === 'undefined') { + logger.warn(`Output format is undefined. Defaulting to "${TEXT_FORMAT}" format`); this.outputFormat = TEXT_FORMAT; - } else if (supportedFormats.includes(opts.outputFormat.toUpperCase())) { - this.outputFormat = opts.outputFormat.toUpperCase(); + } else if (supportedFormats.includes(this.options.outputFormat.toUpperCase())) { + this.outputFormat = this.options.outputFormat.toUpperCase(); + logger.debug(`Output format is set to "${this.outputFormat}" format in worker #${this.workerIndex} in round #${this.roundIndex}`); } else { - logger.warn(`Output format ${opts.outputFormat} is not supported. Defaulting to CSV format`); + logger.warn(`Output format "${this.options.outputFormat}" is not supported. Defaulting to "${TEXT_FORMAT}" format`); this.outputFormat = TEXT_FORMAT; } - this.pathTemplate = opts.pathTemplate; - this.rateController = new RateControl(opts.rateController, clientIdx, roundIdx); + this.pathTemplate = this.options.pathTemplate; + // resolve template path placeholders + this.pathTemplate = this.pathTemplate.replace(//gi, this.roundIndex.toString()); + this.pathTemplate = this.pathTemplate.replace(//gi, this.workerIndex.toString()); + this.pathTemplate = util.resolvePath(this.pathTemplate); + + this.rateController = new RateControl(testMessage, stats, workerIndex,); } /** @@ -119,51 +124,12 @@ class RecordRateController extends RateInterface{ } /** - * Initializes the rate controller. - * - * @param {object} msg Client options with adjusted per-client load settings. - * @param {string} msg.type The type of the message. Currently always 'test' - * @param {string} msg.label The label of the round. - * @param {object} msg.rateControl The rate control to use for the round. - * @param {number} msg.trim The number/seconds of transactions to trim from the results. - * @param {object} msg.args The user supplied arguments for the round. - * @param {string} msg.cb The path of the user's callback module. - * @param {string} msg.config The path of the network's configuration file. - * @param {number} msg.numb The number of transactions to generate during the round. - * @param {number} msg.txDuration The length of the round in SECONDS. - * @param {number} msg.totalClients The number of clients executing the round. - * @param {number} msg.clients The number of clients executing the round. - * @param {object} msg.clientArgs Arguments for the client. - * @param {number} msg.clientIdx The 0-based index of the current client. - * @param {number} msg.roundIdx The 1-based index of the current round. + * Perform the rate control action by blocking the execution for a certain amount of time. * @async */ - async init(msg) { - // if we know the number of transactions beforehand, pre-allocate the array - if (msg.numb) { - this.records = new Array(msg.numb); - this.records.fill(0); - } - - // resolve template path placeholders - this.pathTemplate = this.pathTemplate.replace(//gi, this.roundIdx.toString()); - this.pathTemplate = this.pathTemplate.replace(//gi, this.clientIdx.toString()); - this.pathTemplate = util.resolvePath(this.pathTemplate); - - await this.rateController.init(msg); - } - - /** - * Perform the rate control. - * @param {number} start The epoch time at the start of the round (ms precision). - * @param {number} idx Sequence number of the current transaction. - * @param {object[]} recentResults The list of results of recent transactions. - * @param {object[]} resultStats The aggregated stats of previous results. - * @async - */ - async applyRateControl(start, idx, recentResults, resultStats) { - await this.rateController.applyRateControl(start, idx, recentResults, resultStats); - this.records[idx] = Date.now() - start; + async applyRateControl() { + await this.rateController.applyRateControl(); + this.records[this.stats.getTotalSubmittedTx()] = Date.now() - this.stats.getRoundStartTime(); } /** @@ -189,25 +155,23 @@ class RecordRateController extends RateInterface{ break; } - if (this.logEnd) { - logger.debug(`Recorded Tx submission times for Client#${this.clientIdx} in Round#${this.roundIdx} to ${this.pathTemplate}`); - } + logger.debug(`Recorded Tx submission times for worker #${this.workerIndex} in round #${this.roundIndex} to ${this.pathTemplate}`); } catch (err) { - logger.error(`An error occurred while writing records to ${this.pathTemplate}: ${err.stack ? err.stack : err}`); + logger.error(`An error occurred for worker #${this.workerIndex} in round #${this.roundIndex} while writing records to ${this.pathTemplate}: ${err.stack || err}`); } } } /** - * Creates a new rate controller instance. - * @constructor - * @param {object} opts The rate controller options. - * @param {number} clientIdx The 0-based index of the client who instantiates the controller. - * @param {number} roundIdx The 1-based index of the round the controller is instantiated in. - * @return {RateInterface} The rate controller instance. + * Factory for creating a new rate controller instance. + * @param {TestMessage} testMessage start test message + * @param {TransactionStatisticsCollector} stats The TX stats collector instance. + * @param {number} workerIndex The 0-based index of the worker node. + * + * @return {RateInterface} The new rate controller instance. */ -function createRateController(opts, clientIdx, roundIdx) { - return new RecordRateController(opts, clientIdx, roundIdx); +function createRateController(testMessage, stats, workerIndex) { + return new RecordRateController(testMessage, stats, workerIndex); } module.exports.createRateController = createRateController; diff --git a/packages/caliper-core/lib/worker/rate-control/replayRate.js b/packages/caliper-core/lib/worker/rate-control/replayRate.js index ee309a9900..0c342fcfc5 100644 --- a/packages/caliper-core/lib/worker/rate-control/replayRate.js +++ b/packages/caliper-core/lib/worker/rate-control/replayRate.js @@ -17,7 +17,7 @@ const RateInterface = require('./rateInterface.js'); const fs = require('fs'); const util = require('../../common/utils/caliper-utils'); -const logger = util.getLogger('replayRate.js'); +const logger = util.getLogger('replay-rate-controller'); const TEXT_FORMAT = 'TEXT'; const BINARY_BE_FORMAT = 'BIN_BE'; @@ -27,48 +27,66 @@ const supportedFormats = [TEXT_FORMAT, BINARY_BE_FORMAT, BINARY_LE_FORMAT]; /** * Rate controller for replaying a previously recorded transaction trace. * - * @property {object} options The user-supplied options for the controller. - * @property {object[]} records The record of times for submitted transactions. + * @property {number[]} records The record of relative times for submitted transactions. * @property {string} pathTemplate The template path for the file to record to. - * @property {number} roundIdx The index of the current round. - * @property {number} clientIdx The index of the current client. * @property {string} inputFormat Specifies the input format for the recordings. * @property {number} defaultSleepTime The default sleep time between extra transactions. * @property {string} delimiter The delimiter character for the CSV format. - * @property {boolean} logWarnings Indicates whether to log extra transaction warnings. + * @property {boolean} loggedWarning Indicates whether the warning has been logged for running out of the trace. + * + * @extends RateInterface */ -class ReplayRateController extends RateInterface{ +class ReplayRateController extends RateInterface { /** - * Creates a new instance of the {ReplayRateController} class. - * @constructor - * @param {object} opts Options for the rate controller. - * @param {number} clientIdx The 0-based index of the client who instantiates the controller. - * @param {number} roundIdx The 1-based index of the round the controller is instantiated in. + * Initializes the rate controller instance. + * @param {TestMessage} testMessage start test message + * @param {TransactionStatisticsCollector} stats The TX stats collector instance. + * @param {number} workerIndex The 0-based index of the worker node. */ - constructor(opts, clientIdx, roundIdx) { - super(opts); - this.roundIdx = roundIdx; - this.clientIdx = clientIdx + 1; + constructor(testMessage, stats, workerIndex) { + super(testMessage, stats, workerIndex); this.records = []; - if (typeof opts.pathTemplate === 'undefined') { + if (typeof this.options.pathTemplate === 'undefined') { throw new Error('The path to load the recording from is undefined'); } - this.pathTemplate = opts.pathTemplate; - this.logWarnings = Boolean(opts.logWarnings || false); - this.defaultSleepTime = Number(opts.defaultSleepTime || 20); + this.loggedWarning = false; + this.defaultSleepTime = Number(this.options.defaultSleepTime || 100); // check for supported input formats - if (typeof opts.inputFormat === 'undefined') { - logger.warn(`[ReplayRateController] Input format is undefined. Defaulting to ${TEXT_FORMAT} format`); + if (typeof this.options.inputFormat === 'undefined') { + logger.warn(`Input format is undefined. Defaulting to "${TEXT_FORMAT}" format`); this.inputFormat = TEXT_FORMAT; - } else if (supportedFormats.includes(opts.inputFormat.toUpperCase())) { - this.inputFormat = opts.inputFormat.toUpperCase(); + } else if (supportedFormats.includes(this.options.inputFormat.toUpperCase())) { + this.inputFormat = this.options.inputFormat.toUpperCase(); + logger.debug(`Input format is set to "${this.inputFormat}" format in worker #${this.workerIndex} in round #${this.roundIndex}`); } else { - logger.warn(`[ReplayRateController] Input format ${opts.inputFormat} is not supported. Defaulting to CSV format`); + logger.warn(`Input format "${this.options.inputFormat}" is not supported. Defaulting to "${TEXT_FORMAT}" format`); this.inputFormat = TEXT_FORMAT; } + + this.pathTemplate = this.options.pathTemplate; + // resolve template path placeholders + this.pathTemplate = this.pathTemplate.replace(//gi, this.roundIndex.toString()); + this.pathTemplate = this.pathTemplate.replace(//gi, this.workerIndex.toString()); + this.pathTemplate = util.resolvePath(this.pathTemplate); + + if (!fs.existsSync(this.pathTemplate)) { + throw new Error(`Trace file does not exist: ${this.pathTemplate}`); + } + + switch (this.inputFormat) { + case TEXT_FORMAT: + this._importFromText(); + break; + case BINARY_BE_FORMAT: + this._importFromBinaryBigEndian(); + break; + case BINARY_LE_FORMAT: + this._importFromBinaryLittleEndian(); + break; + } } /** @@ -107,67 +125,20 @@ class ReplayRateController extends RateInterface{ } /** - * Initializes the rate controller. - * - * @param {object} msg Client options with adjusted per-client load settings. - * @param {string} msg.type The type of the message. Currently always 'test' - * @param {string} msg.label The label of the round. - * @param {object} msg.rateControl The rate control to use for the round. - * @param {number} msg.trim The number/seconds of transactions to trim from the results. - * @param {object} msg.args The user supplied arguments for the round. - * @param {string} msg.cb The path of the user's callback module. - * @param {string} msg.config The path of the network's configuration file. - * @param {number} msg.numb The number of transactions to generate during the round. - * @param {number} msg.txDuration The length of the round in SECONDS. - * @param {number} msg.totalClients The number of clients executing the round. - * @param {number} msg.clients The number of clients executing the round. - * @param {object} msg.clientArgs Arguments for the client. - * @param {number} msg.clientIdx The 0-based index of the current client. - * @param {number} msg.roundIdx The 1-based index of the current round. - * @async - */ - async init(msg) { - // resolve template path placeholders - this.pathTemplate = this.pathTemplate.replace(//gi, this.roundIdx.toString()); - this.pathTemplate = this.pathTemplate.replace(//gi, this.clientIdx.toString()); - this.pathTemplate = util.resolvePath(this.pathTemplate); - - if (!fs.existsSync(this.pathTemplate)) { - throw new Error(`Trace file does not exist: ${this.pathTemplate}`); - } - - switch (this.inputFormat) { - case TEXT_FORMAT: - this._importFromText(); - break; - case BINARY_BE_FORMAT: - this._importFromBinaryBigEndian(); - break; - case BINARY_LE_FORMAT: - this._importFromBinaryLittleEndian(); - break; - default: - throw new Error(`Unsupported replay rate controller input format: ${this.inputFormat}`); - } - } - - /** - * Perform the rate control. - * @param {number} start The epoch time at the start of the round (ms precision). - * @param {number} idx Sequence number of the current transaction. - * @param {object[]} recentResults The list of results of recent transactions. - * @param {object[]} resultStats The aggregated stats of previous results. + * Perform the rate control action by blocking the execution for a certain amount of time. * @async */ - async applyRateControl(start, idx, recentResults, resultStats) { - if (idx <= this.records.length - 1) { - let sleepTime = this.records[idx] - (Date.now() - start); + async applyRateControl() { + let currentIndex = this.stats.getTotalSubmittedTx(); + if (currentIndex <= this.records.length - 1) { + let sleepTime = this.records[currentIndex] - (Date.now() - this.stats.getRoundStartTime()); if (sleepTime > 5) { await util.sleep(sleepTime); } } else { - if (this.logWarnings) { - logger.warn(`Using default sleep time of ${this.defaultSleepTime}ms for Tx#${idx}`); + if (!this.loggedWarning) { + logger.warn(`Using default sleep time of ${this.defaultSleepTime} ms from now on for worker #${this.workerIndex} in round #${this.roundIndex}`); + this.loggedWarning = true; } await util.sleep(this.defaultSleepTime); } @@ -181,15 +152,15 @@ class ReplayRateController extends RateInterface{ } /** - * Creates a new rate controller instance. - * @constructor - * @param {object} opts The rate controller options. - * @param {number} clientIdx The 0-based index of the client who instantiates the controller. - * @param {number} roundIdx The 1-based index of the round the controller is instantiated in. - * @return {RateInterface} The rate controller instance. + * Factory for creating a new rate controller instance. + * @param {TestMessage} testMessage start test message + * @param {TransactionStatisticsCollector} stats The TX stats collector instance. + * @param {number} workerIndex The 0-based index of the worker node. + * + * @return {RateInterface} The new rate controller instance. */ -function createRateController(opts, clientIdx, roundIdx) { - return new ReplayRateController(opts); +function createRateController(testMessage, stats, workerIndex) { + return new ReplayRateController(testMessage, stats, workerIndex); } module.exports.createRateController = createRateController; diff --git a/packages/caliper-core/lib/worker/tx-observers/internal-tx-observer.js b/packages/caliper-core/lib/worker/tx-observers/internal-tx-observer.js new file mode 100644 index 0000000000..e4eb2dabe5 --- /dev/null +++ b/packages/caliper-core/lib/worker/tx-observers/internal-tx-observer.js @@ -0,0 +1,81 @@ +/* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +'use strict'; + +const TxObserverInterface = require('./tx-observer-interface'); +const TxUpdateMessage = require('../../common/messages/txUpdateMessage'); +const TxResetMessage = require('../../common/messages/txResetMessage'); +const CaliperUtils = require('../../common/utils/caliper-utils'); +const ConfigUtil = require('../../common/config/config-util'); + +/** + * Internal TX observer used by the worker process to driver TX scheduling and report round statistics + * It is always instantiated. + */ +class InternalTxObserver extends TxObserverInterface{ + /** + * Initializes the observer instance. + * @param {WorkerMessageHandler} workerMessageHandler The worker message handler instance. Not used. + * @param {string} managerUuid The UUID of the messenger for message sending. + */ + constructor(workerMessageHandler, managerUuid) { + super(workerMessageHandler); + this.updateInterval = ConfigUtil.get(ConfigUtil.keys.Worker.Update.Interval); + this.intervalObject = undefined; + this.messengerUUID = workerMessageHandler.getUUID(); + this.managerUuid = managerUuid; + } + + /** + * Sends the current aggregated statistics to the master node when triggered by "setInterval". + * @private + */ + async _sendUpdate() { + let txUpdateMessage = new TxUpdateMessage(this.messengerUUID, [this.managerUuid], super.getCurrentStatistics()); + await this.workerMessageHandler.send(txUpdateMessage); + } + + /** + * Activates the TX observer instance and starts the regular update scheduling. + * @param {number} workerIndex The 0-based index of the worker node. + * @param {number} roundIndex The 0-based index of the current round. + * @param {string} roundLabel The roundLabel name. + */ + async activate(workerIndex, roundIndex, roundLabel) { + await super.activate(workerIndex, roundIndex, roundLabel); + this.intervalObject = setInterval(async () => { await this._sendUpdate(); }, this.updateInterval); + } + + /** + * Deactivates the TX observer interface, and stops the regular update scheduling. + */ + async deactivate() { + await super.deactivate(); + + if (this.intervalObject) { + clearInterval(this.intervalObject); + + await this._sendUpdate(); + await CaliperUtils.sleep(this.updateInterval); + + // TODO: the txResult message should be enough + // or the round-end message should include the final stats + let txResetMessage = new TxResetMessage(this.messengerUUID, [this.managerUuid]); + await this.workerMessageHandler.send(txResetMessage); + } + } +} + +module.exports = InternalTxObserver; diff --git a/packages/caliper-core/lib/worker/tx-observers/logging-tx-observer.js b/packages/caliper-core/lib/worker/tx-observers/logging-tx-observer.js new file mode 100644 index 0000000000..01568aca97 --- /dev/null +++ b/packages/caliper-core/lib/worker/tx-observers/logging-tx-observer.js @@ -0,0 +1,79 @@ +/* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +'use strict'; + +const TxObserverInterface = require('./tx-observer-interface'); +const CaliperUtils = require('../../common/utils/caliper-utils'); + +/** + * TX observer used to log every TX result to stdout in JSON format. + * + * @property {function} logFunction The configured logging function. + */ +class LoggingTxObserver extends TxObserverInterface{ + /** + * Initializes the observer instance. + * @param {object} options The log observer configuration object. + * @param {WorkerMessageHandler} workerMessageHandler The worker message handler instance. Not used. + */ + constructor(options, workerMessageHandler) { + super(workerMessageHandler); + let logger = CaliperUtils.getLogger(options.loggerModuleName || 'txinfo'); + this.logFunction = logger[options.messageLevel || 'info']; + } + + /** + * Called when TXs are submitted. The observer ignores this event + * @param {number} count The number of submitted TXs. Can be greater than one for a batch of TXs. + */ + txSubmitted(count) { } + + /** + * Called when TXs are finished. The observer logs the results in a stateless way (i.e., does not process them further). + * @param {TxStatus | TxStatus[]} results The result information of the finished TXs. Can be a collection of results for a batch of TXs. + */ + txFinished(results) { + // TODO: appending metadata should be done by the dispatch + if (Array.isArray(results)) { + for (let result of results) { + // add extra metadata + result.workerIndex = this.workerIndex; + result.roundIndex = this.roundIndex; + + // TODO: use fast-json-stringify + // this.logFunction(JSON.stringify(result)); + } + } else { + // add extra metadata + results.workerIndex = this.workerIndex; + results.roundIndex = this.roundIndex; + + // TODO: use fast-json-stringify + // this.logFunction(JSON.stringify(results)); + } + } +} + +/** + * Factory function for creating a LoggingTxObserver instance. + * @param {object} options The logging observer configuration object. + * @param {WorkerMessageHandler} workerMessageHandler The worker message handler instance. Not used. + * @return {TxObserverInterface} The observer instance. + */ +function createTxObserver(options, workerMessageHandler) { + return new LoggingTxObserver(options, workerMessageHandler); +} + +module.exports.createTxObserver = createTxObserver; diff --git a/packages/caliper-core/lib/worker/tx-observers/prometheus-push-tx-observer.js b/packages/caliper-core/lib/worker/tx-observers/prometheus-push-tx-observer.js new file mode 100644 index 0000000000..ea5920c25e --- /dev/null +++ b/packages/caliper-core/lib/worker/tx-observers/prometheus-push-tx-observer.js @@ -0,0 +1,121 @@ +/* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +'use strict'; + +const TxObserverInterface = require('./tx-observer-interface'); +// const TxResetMessage = require('../../common/messages/txResetMessage'); +const PrometheusClient = require('../../common/prometheus/prometheus-push-client'); +const CaliperUtils = require('../../common/utils/caliper-utils'); + +/** + * Prometheus TX observer used to maintain Prometheus metrics for the push-based scenario (through a push gateway). + */ +class PrometheusPushTxObserver extends TxObserverInterface { + /** + * Initializes the observer instance. + * @param {object} options The observer configuration object. + * @param {WorkerMessageHandler} workerMessageHandler The worker message handler instance. + */ + constructor(options, workerMessageHandler) { + super(workerMessageHandler); + this.sendInterval = options && options.sendInterval || 1000; + this.intervalObject = undefined; + + this.prometheusClient = new PrometheusClient(); + this.prometheusClient.setGateway(options.push_url); + + this.internalStats = { + previouslyCompletedTotal: 0, + previouslySubmittedTotal: 0 + }; + } + + /** + * Sends the current aggregated statistics to the master node when triggered by "setInterval". + * @private + */ + async _sendUpdate() { + const stats = super.getCurrentStatistics(); + this.prometheusClient.configureTarget(stats.getRoundLabel(), stats.getRoundIndex(), stats.getWorkerIndex()); + + // Observer based requirements + this.prometheusClient.push('caliper_txn_success', stats.getTotalSuccessfulTx()); + this.prometheusClient.push('caliper_txn_failure', stats.getTotalFailedTx()); + this.prometheusClient.push('caliper_txn_pending', stats.getTotalSubmittedTx() - stats.getTotalFinishedTx()); + + // TxStats based requirements, existing behaviour batches results bounded within txUpdateTime + const completedTransactions = stats.getTotalSuccessfulTx() + stats.getTotalFailedTx(); + const submittedTransactions = stats.getTotalSubmittedTx(); + + const batchCompletedTransactions = completedTransactions - this.internalStats.previouslyCompletedTotal; + const batchTPS = (batchCompletedTransactions/this.sendInterval)*1000; // txUpdate is in ms + + const batchSubmittedTransactions = submittedTransactions - this.internalStats.previouslyCompletedTotal; + const batchSubmitTPS = (batchSubmittedTransactions/this.sendInterval)*1000; // txUpdate is in ms + const latency = (stats.getTotalLatencyForFailed() + stats.getTotalLatencyForSuccessful()) / completedTransactions; + + this.prometheusClient.push('caliper_tps', batchTPS); + this.prometheusClient.push('caliper_latency', latency/1000); + this.prometheusClient.push('caliper_txn_submit_rate', batchSubmitTPS); + + this.internalStats.previouslyCompletedTotal = batchCompletedTransactions; + this.internalStats.previouslyCompletedTotal = batchSubmittedTransactions; + } + + /** + * Activates the TX observer instance and starts the regular update scheduling. + * @param {number} workerIndex The 0-based index of the worker node. + * @param {number} roundIndex The 0-based index of the current round. + * @param {string} roundLabel The roundLabel name. + */ + async activate(workerIndex, roundIndex, roundLabel) { + await super.activate(workerIndex, roundIndex, roundLabel); + this.intervalObject = setInterval(async () => { await this._sendUpdate(); }, this.sendInterval); + } + + /** + * Deactivates the TX observer interface, and stops the regular update scheduling. + */ + async deactivate() { + await super.deactivate(); + + this.internalStats = { + previouslyCompletedTotal: 0, + previouslySubmittedTotal: 0 + }; + + if (this.intervalObject) { + clearInterval(this.intervalObject); + + this.prometheusClient.push('caliper_txn_success', 0); + this.prometheusClient.push('caliper_txn_failure', 0); + this.prometheusClient.push('caliper_txn_pending', 0); + await CaliperUtils.sleep(this.sendInterval); + } + } + +} + +/** + * Factory function for creating a PrometheusPushTxObserver instance. + * @param {object} options The observer configuration object. + * @param {WorkerMessageHandler} workerMessageHandler The worker message handler instance. + * @return {TxObserverInterface} The observer instance. + */ +function createTxObserver(options, workerMessageHandler) { + return new PrometheusPushTxObserver(options, workerMessageHandler); +} + +module.exports.createTxObserver = createTxObserver; diff --git a/packages/caliper-core/lib/worker/tx-observers/tx-observer-dispatch.js b/packages/caliper-core/lib/worker/tx-observers/tx-observer-dispatch.js new file mode 100644 index 0000000000..6746c8b54e --- /dev/null +++ b/packages/caliper-core/lib/worker/tx-observers/tx-observer-dispatch.js @@ -0,0 +1,134 @@ +/* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +'use strict'; + +const path = require('path'); +const TxObserverInterface = require('./tx-observer-interface'); +const CaliperUtils = require('../../common/utils/caliper-utils'); + +const builtInTxObservers = new Map([ + ['logging', path.join(__dirname, 'logging-tx-observer.js')], + // spoilers :) ['prometheus', path.join(__dirname, 'prometheus-tx-observer.js')], + ['prometheus-push', path.join(__dirname, 'prometheus-push-tx-observer.js')] +]); + +/** + * TX observer dispatch to manage the broadcast of TX events to other TX observers. + * + * @property {TxObserverInterface[]} txObservers Collection of configured TX observers. + */ +class TxObserverDispatch extends TxObserverInterface { + /** + * Initializes the TX observer dispatch instance. + * @param {WorkerMessageHandler} workerMessageHandler The worker message handler instance. + * @param {TxObserverInterface} internalTxObserver The executor's internal TX observer instance. + * @param {string} managerUuid The UUID of the messenger for message sending. + */ + constructor(workerMessageHandler, internalTxObserver, managerUuid) { + super(workerMessageHandler); + // contains the loaded TX observers + this.txObservers = []; + + // load the configured TX observers + let observerConfigs = super.getDeclaredTxObservers(); + for (let observer of observerConfigs) { + const factoryFunction = CaliperUtils.loadModuleFunction(builtInTxObservers, observer.module, 'createTxObserver'); + this.txObservers.push(factoryFunction(observer.options, workerMessageHandler, managerUuid)); + } + + // always load the internal TX observer + this.txObservers.push(internalTxObserver); + } + + /** + * Activates the dispatch, and in turn, every configured TX observer instance. + * @param {number} workerIndex The 0-based index of the worker node. + * @param {number} roundIndex The 0-based index of the current round. + * @param {string} roundLabel The roundLabel name. + * @async + */ + async activate(workerIndex, roundIndex, roundLabel) { + await super.activate(workerIndex, roundIndex, roundLabel); + + for (let observer of this.txObservers) { + await observer.activate(workerIndex, roundIndex, roundLabel); + } + } + + /** + * Deactivates the dispatch, and in turn, every configured TX observer instance. + * @async + */ + async deactivate() { + await super.deactivate(); + + for (let observer of this.txObservers) { + await observer.deactivate(); + } + } + + /** + * Called when TXs are submitted. The dispatch forwards the event to every configured TX observer instance. + * @param {number} count The number of submitted TXs. Can be greater than one for a batch of TXs. + */ + txSubmitted(count) { + if (!this.active) { + return; + } + + for (let observer of this.txObservers) { + observer.txSubmitted(count); + } + } + + /** + * Called when TXs are finished. The dispatch forwards the event to every configured TX observer instance. + * @param {TxStatus | TxStatus[]} results The result information of the finished TXs. Can be a collection of results for a batch of TXs. + */ + txFinished(results) { + if (!this.active) { + return; + } + + for (let observer of this.txObservers) { + observer.txFinished(results); + } + } +} + +module.exports = TxObserverDispatch; + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packages/caliper-core/lib/worker/tx-observers/tx-observer-interface.js b/packages/caliper-core/lib/worker/tx-observers/tx-observer-interface.js new file mode 100644 index 0000000000..937190a0c4 --- /dev/null +++ b/packages/caliper-core/lib/worker/tx-observers/tx-observer-interface.js @@ -0,0 +1,104 @@ +/* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +'use strict'; + +const CaliperUtils = require('../../common/utils/caliper-utils'); +const ConfigUtil = require('../../common/config/config-util.js'); +const TransactionStatisticsCollector = require('../../common/core/transaction-statistics-collector'); + +/** + * Interface of TX observers. + * + * @property {boolean} active Indicates whether the TX observer is active or not. + * @property {number} workerIndex The 0-based index of the worker node. + * @property {number} currentRound The 0-based index of the current round. + * @property {TransactionStatisticsCollector[]} Collection of TX statistics corresponding to each round. + * @property {WorkerMessageHandler} workerMessageHandler The worker message handler instance. + */ +class TxObserverInterface { + /** + * Initializes the TX observer instance. + * @param {WorkerMessageHandler} workerMessageHandler The worker message handler instance. + */ + constructor(workerMessageHandler) { + this.workerMessageHandler = workerMessageHandler; + this.active = false; + this.workerIndex = 0; + this.currentRound = 0; + this.roundStatistics = []; + + // config path + this.benchmarkConfigPath = CaliperUtils.resolvePath(ConfigUtil.get(ConfigUtil.keys.BenchConfig)); + + // config object + const benchmarkConfig = CaliperUtils.parseYaml(this.benchmarkConfigPath); + this.observerConfig = benchmarkConfig.monitors && benchmarkConfig.monitors.transaction ? benchmarkConfig.monitors.transaction : []; + } + + /** + * Return an array of declared txObservers + * @return {string[]} An array of declared txObservers + */ + getDeclaredTxObservers() { + return this.observerConfig; + } + + /** + * Activates the TX observer instance, and in turn, the new TX statistics collector. + * @param {number} workerIndex The 0-based index of the worker node. + * @param {number} roundIndex The 0-based index of the current round. + * @param {string} roundLabel The roundLabel name. + */ + async activate(workerIndex, roundIndex, roundLabel) { + this.active = true; + this.workerIndex = workerIndex; + this.currentRound = roundIndex; + this.roundStatistics[this.currentRound] = new TransactionStatisticsCollector(workerIndex, roundIndex, roundLabel); + this.roundStatistics[this.currentRound].activate(); + } + + /** + * Deactivates the TX observer interface, and in turn, the current TX statistics collector. */ + async deactivate() { + this.active = false; + this.roundStatistics[this.currentRound].deactivate(); + } + + /** + * Called when TXs are submitted. + * @param {number} count The number of submitted TXs. Can be greater than one for a batch of TXs. + */ + txSubmitted(count) { + this.roundStatistics[this.currentRound].txSubmitted(count); + } + + /** + * Called when TXs are finished. + * @param {TxStatus | TxStatus[]} results The result information of the finished TXs. Can be a collection of results for a batch of TXs. + */ + txFinished(results) { + this.roundStatistics[this.currentRound].txFinished(results); + } + + /** + * Return the underlying TX statistics collector. + * @return {TransactionStatisticsCollector} The TX statistics collector instance. + */ + getCurrentStatistics() { + return this.roundStatistics[this.currentRound]; + } +} + +module.exports = TxObserverInterface; diff --git a/packages/caliper-core/lib/worker/message-handler.js b/packages/caliper-core/lib/worker/worker-message-handler.js similarity index 93% rename from packages/caliper-core/lib/worker/message-handler.js rename to packages/caliper-core/lib/worker/worker-message-handler.js index 585a369918..dea51d69e8 100644 --- a/packages/caliper-core/lib/worker/message-handler.js +++ b/packages/caliper-core/lib/worker/worker-message-handler.js @@ -15,13 +15,13 @@ 'use strict'; const CaliperWorker = require('./caliper-worker'); -const MessageTypes = require('./../common/utils/constants').Messages.Types; +const MessageTypes = require('../common/utils/constants').Messages.Types; -const ConnectedMessage = require('./../common/messages/connectedMessage'); -const AssignedMessage = require('./../common/messages/assignedMessage'); -const ReadyMessage = require('./../common/messages/readyMessage'); -const PreparedMessage = require('./../common/messages/preparedMessage'); -const TestResultMessage = require('./../common/messages/testResultMessage'); +const ConnectedMessage = require('../common/messages/connectedMessage'); +const AssignedMessage = require('../common/messages/assignedMessage'); +const ReadyMessage = require('../common/messages/readyMessage'); +const PreparedMessage = require('../common/messages/preparedMessage'); +const TestResultMessage = require('../common/messages/testResultMessage'); const logger = require('../common/utils/caliper-utils.js').getLogger('worker-message-handler'); @@ -36,7 +36,7 @@ const logger = require('../common/utils/caliper-utils.js').getLogger('worker-mes * @property {BlockchainConnector} The SUT connector instance. * @property {{resolve, reject}} exitPromiseFunctions The resolve/reject Promise functions to handle the async exit message later. */ -class MessageHandler { +class WorkerMessageHandler { /** * Initializes the message handler instance. * @param {MessengerInterface} messenger The messenger to use for communication with the manager. @@ -108,7 +108,7 @@ class MessageHandler { ? `Worker#${this.workerIndex} (${this.messenger.getUUID()})` : `Worker (${this.messenger.getUUID()})`; - logger.debug(`Handling "${message.getType()}" message for ${workerID}: ${message.stringify()}`); + logger.info(`Handling "${message.getType()}" message for ${workerID}: ${message.stringify()}`); } /** @@ -232,7 +232,7 @@ class MessageHandler { let err; let result; try { - result = await this.worker.doTest(message); + result = await this.worker.executeRound(message); logger.info(`Worker#${this.workerIndex} finished Round#${message.getRoundIndex()}`); } catch (error) { err = error; @@ -266,4 +266,4 @@ class MessageHandler { } } -module.exports = MessageHandler; +module.exports = WorkerMessageHandler; diff --git a/packages/caliper-core/test/manager/monitors/monitor-prometheus.js b/packages/caliper-core/test/manager/monitors/monitor-prometheus.js index e27dd95d29..3810fd3519 100644 --- a/packages/caliper-core/test/manager/monitors/monitor-prometheus.js +++ b/packages/caliper-core/test/manager/monitors/monitor-prometheus.js @@ -23,9 +23,7 @@ const sinon = require('sinon'); describe('Prometheus monitor implementation', () => { - const fakePushClient = sinon.stub(); const fakeQueryClient = sinon.stub(); - PrometheusMonitorRewire.__set__('PrometheusPushClient', fakePushClient); PrometheusMonitorRewire.__set__('PrometheusQueryClient', fakeQueryClient); // Before/After @@ -89,16 +87,6 @@ describe('Prometheus monitor implementation', () => { }); }); - describe('#getPushClient', () => { - - it('should return the internal Push Client', () => { - const mon = new PrometheusMonitorRewire({}); - const test = 'penguin'; - mon.prometheusPushClient = test; - mon.getPushClient().should.equal(test); - }); - }); - describe('#getQueryClient', ()=>{ it('should return the internal Query Client', () => { @@ -109,14 +97,6 @@ describe('Prometheus monitor implementation', () => { }); }); - describe('#getPushGatewayURL', () => { - - it('should return the push gateway url from teh config file', () => { - const mon = new PrometheusMonitorRewire({push_url: '123'}); - mon.getPushGatewayURL().should.equal('123'); - }); - }); - describe('#start', () => { it('should set the start time with the current time', () => { diff --git a/packages/caliper-core/test/manager/orchestrators/worker-orchestrator.js b/packages/caliper-core/test/manager/orchestrators/worker-orchestrator.js index 7a920ee80c..5b6707e7c7 100644 --- a/packages/caliper-core/test/manager/orchestrators/worker-orchestrator.js +++ b/packages/caliper-core/test/manager/orchestrators/worker-orchestrator.js @@ -16,6 +16,7 @@ const rewire = require('rewire'); const WorkerOrchestratorRewire = rewire('../../../lib/manager/orchestrators/worker-orchestrator'); +const TransactionStatisticsCollector = require('../../../lib/common/core/transaction-statistics-collector'); const chai = require('chai'); chai.should(); @@ -102,7 +103,7 @@ describe('worker orchestrator implementation', () => { const checkVal = 'this is my update'; // overwrite with known value myOrchestrator.updates = checkVal; - // assert repsonse + // assert response myOrchestrator.getUpdates().should.equal(checkVal); }); @@ -111,33 +112,39 @@ describe('worker orchestrator implementation', () => { describe('#formatResults', () => { const myOrchestrator = new WorkerOrchestratorRewire(benchmarkConfig, workerFactory); - it('should group all worker results into an array under a results label', () => { - const result0 = {results: [1] , start: new Date(2018, 11, 24, 10, 33), end: new Date(2018, 11, 24, 11, 33)}; - const result1 = {results: [2] , start: new Date(2018, 11, 24, 10, 34), end: new Date(2018, 11, 24, 11, 23)}; - const result2 = {results: [3] , start: new Date(2018, 11, 24, 10, 35), end: new Date(2018, 11, 24, 11, 13)}; - const testData = [result0, result1, result2]; + let workerStats0; + let workerStats1; + beforeEach( () => { + workerStats0 = new TransactionStatisticsCollector(0, 0, 'testLabel'); + workerStats1 = new TransactionStatisticsCollector(1, 0, 'testLabel'); + workerStats0.activate(); + workerStats1.activate(); + }); + + it('should merge all worker results into a single txStats object', () => { + workerStats0.txSubmitted(3); + workerStats1.txSubmitted(2); + const testData = [workerStats0, workerStats1]; const output = myOrchestrator.formatResults(testData); - output.results.should.deep.equal([1,2,3]); + output.results.getTotalSubmittedTx().should.equal(5); }); it('should determine and persist the time when all workers have started', () => { - const compareStart = new Date(2018, 11, 24, 10, 35); - const result0 = {results: [1] , start: new Date(2018, 11, 24, 10, 33), end: new Date(2018, 11, 24, 11, 33)}; - const result1 = {results: [2] , start: new Date(2018, 11, 24, 10, 34), end: new Date(2018, 11, 24, 11, 13)}; - const result2 = {results: [3] , start: compareStart, end: new Date(2018, 11, 24, 11, 23)}; - const testData = [result0, result1, result2]; + const compareStart = Date.UTC(2018, 11, 24); + workerStats0.stats.metadata.roundStartTime = Date.UTC(2018, 11, 24); + workerStats1.stats.metadata.roundStartTime = Date.UTC(2018, 11, 24); + const testData = [workerStats0, workerStats1]; const output = myOrchestrator.formatResults(testData); output.start.should.equal(compareStart); }); it('should determine and persist the last time when all workers were running', () => { - const compareEnd = new Date(2018, 11, 24, 11, 13); - const result0 = {results: [1] , start: new Date(2018, 11, 24, 10, 33), end: new Date(2018, 11, 24, 11, 33)}; - const result1 = {results: [2] , start: new Date(2018, 11, 24, 10, 34), end: compareEnd}; - const result2 = {results: [3] , start: new Date(2018, 11, 24, 10, 35), end: new Date(2018, 11, 24, 11, 23)}; - const testData = [result0, result1, result2]; + const compareEnd = Date.UTC(2018, 11, 24); + workerStats0.stats.metadata.roundFinishTime = Date.UTC(2018, 11, 24); + workerStats1.stats.metadata.roundFinishTime = Date.UTC(2018, 11, 24); + const testData = [workerStats0, workerStats1]; const output = myOrchestrator.formatResults(testData); output.end.should.equal(compareEnd); diff --git a/packages/caliper-core/test/manager/report/report.js b/packages/caliper-core/test/manager/report/report.js index be1a235931..4a00fb2079 100644 --- a/packages/caliper-core/test/manager/report/report.js +++ b/packages/caliper-core/test/manager/report/report.js @@ -15,6 +15,7 @@ 'use strict'; const Report = require('../../../lib/manager/report/report'); +const TransactionStatisticsCollector = require('../../../lib/common/core/transaction-statistics-collector'); const chai = require('chai'); chai.should(); @@ -22,117 +23,117 @@ const sinon = require('sinon'); describe('report implementation', () => { - describe('#getLocalResultValues', () => { + describe('#getResultValues', () => { it('should retrieve a result column map', () => { const report = new Report(); + const txnStats = new TransactionStatisticsCollector(); const getResultColumnMapSpy = new sinon.stub().returns(new Map()); report.getResultColumnMap = getResultColumnMapSpy; - report.getLocalResultValues('label', {}); + report.getResultValues('label', txnStats); sinon.assert.calledOnce(getResultColumnMapSpy); }); it('should set Name to `unknown` if missing', () => { const report = new Report(); - const output = report.getLocalResultValues(null, {}); + const txnStats = new TransactionStatisticsCollector(); + + const output = report.getResultValues(null, txnStats); output.get('Name').should.equal('unknown'); }); it('should set Name if available', () => { const report = new Report(); - const output = report.getLocalResultValues('myTestLabel', {}); - output.get('Name').should.equal('myTestLabel'); - }); + const txnStats = new TransactionStatisticsCollector(); - it('should set Succ to `-` if missing', () => { - const report = new Report(); - const output = report.getLocalResultValues('myTestLabel', {}); - output.get('Succ').should.equal('-'); + const output = report.getResultValues('myTestLabel', txnStats); + output.get('Name').should.equal('myTestLabel'); }); it('should set Succ if available', () => { const report = new Report(); - const output = report.getLocalResultValues('myTestLabel', {succ: '42'}); - output.get('Succ').should.equal('42'); + const txnStats = new TransactionStatisticsCollector(); + txnStats.stats.txCounters.totalSuccessful = 42; + + const output = report.getResultValues('myTestLabel', txnStats); + output.get('Succ').should.equal(42); }); it('should set Succ to zero if passed', () => { const report = new Report(); - const output = report.getLocalResultValues('myTestLabel', {succ: '0'}); - output.get('Succ').should.equal('0'); - }); + const txnStats = new TransactionStatisticsCollector(); - it('should set Fail to `-` if missing', () => { - const report = new Report(); - const output = report.getLocalResultValues('myTestLabel', {}); - output.get('Fail').should.equal('-'); + const output = report.getResultValues('myTestLabel', txnStats); + output.get('Succ').should.equal(0); }); it('should set Fail if available', () => { const report = new Report(); - const output = report.getLocalResultValues('myTestLabel', {fail: '38'}); - output.get('Fail').should.equal('38'); - }); + const txnStats = new TransactionStatisticsCollector(); + txnStats.stats.txCounters.totalFailed = 38; - it('should set Max Latency to `-` if missing', () => { - const report = new Report(); - const output = report.getLocalResultValues('myTestLabel', {}); - output.get('Max Latency (s)').should.equal('-'); + const output = report.getResultValues('myTestLabel', txnStats); + output.get('Fail').should.equal(38); }); it('should set Max Latency to 2DP if available', () => { const report = new Report(); - const output = report.getLocalResultValues('myTestLabel', {delay: { max: 1.2322}} ); + const txnStats = new TransactionStatisticsCollector(); + txnStats.stats.latency.successful.max = 1232.2; + + const output = report.getResultValues('myTestLabel', txnStats ); output.get('Max Latency (s)').should.equal('1.23'); }); - it('should set Min Latency to `-` if missing', () => { - const report = new Report(); - const output = report.getLocalResultValues('myTestLabel', {}); - output.get('Min Latency (s)').should.equal('-'); - }); it('should set Min Latency to 2DP if available', () => { const report = new Report(); - const output = report.getLocalResultValues('myTestLabel', {delay: { min: 0.2322}}); + const txnStats = new TransactionStatisticsCollector(); + txnStats.stats.latency.successful.min = 232.2; + + const output = report.getResultValues('myTestLabel', txnStats); output.get('Min Latency (s)').should.equal('0.23'); }); - it('should set Avg Latency to `-` if missing', () => { + it('should set Avg Latency to `-` if no successful transactions', () => { const report = new Report(); - const output = report.getLocalResultValues('myTestLabel', {}); + const txnStats = new TransactionStatisticsCollector(); + + const output = report.getResultValues('myTestLabel', txnStats); output.get('Avg Latency (s)').should.equal('-'); }); it('should set Avg Latency to 2DP if available', () => { const report = new Report(); - const output = report.getLocalResultValues('myTestLabel', { succ: 3, delay: { sum: 10}}); - output.get('Avg Latency (s)').should.equal('3.33'); - }); + const txnStats = new TransactionStatisticsCollector(); + txnStats.stats.txCounters.totalSuccessful = 3; + txnStats.stats.latency.successful.total = 10000; - it('should set Send Rate `-` if missing', () => { - const report = new Report(); - const output = report.getLocalResultValues('myTestLabel', {}); - output.get('Send Rate (TPS)').should.equal('-'); + const output = report.getResultValues('myTestLabel', txnStats); + output.get('Avg Latency (s)').should.equal('3.33'); }); it('should set Send Rate to 1DP if available', () => { const report = new Report(); - const output = report.getLocalResultValues('myTestLabel', { succ:500, fail:0, create:{min:1565001755.094, max:1565001774.893} }); - output.get('Send Rate (TPS)').should.equal('25.3'); - }); + const txnStats = new TransactionStatisticsCollector(); + txnStats.stats.txCounters.totalSuccessful = 500; + txnStats.stats.timestamps.firstCreateTime = 1565001755094; + txnStats.stats.timestamps.lastCreateTime = 1565001774893; - it('should set Throughput to `-` if missing', () => { - const report = new Report(); - const output = report.getLocalResultValues('myTestLabel', {}); - output.get('Throughput (TPS)').should.equal('-'); + const output = report.getResultValues('myTestLabel', txnStats); + output.get('Send Rate (TPS)').should.equal('25.3'); }); it('should set Throughput to 1DP if available', () => { const report = new Report(); - const output = report.getLocalResultValues('myTestLabel', {succ:500,fail:0,create:{min:1565001755.094,max:1565001774.893},final:{min:1565001755.407,max:1565001774.988,last:1565001774.988},delay:{min:0.072,max:0.342,sum:98.64099999999999,detail:[]},out:[],sTPTotal:0,sTTotal:0,invokeTotal:0,length:500}); + const txnStats = new TransactionStatisticsCollector(); + txnStats.stats.txCounters.totalSuccessful = 500; + txnStats.stats.timestamps.lastFinishTime = 1565001774988; + txnStats.stats.timestamps.firstCreateTime = 1565001755094; + + const output = report.getResultValues('myTestLabel', txnStats); output.get('Throughput (TPS)').should.equal('25.1'); }); }); diff --git a/packages/caliper-core/test/worker/rate-control/fixedBacklog.js b/packages/caliper-core/test/worker/rate-control/fixedBacklog.js deleted file mode 100644 index b9a56b0e98..0000000000 --- a/packages/caliper-core/test/worker/rate-control/fixedBacklog.js +++ /dev/null @@ -1,225 +0,0 @@ -/* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -'use strict'; - -const rewire = require('rewire'); -const FixedBacklog = rewire('../../../lib/worker/rate-control/fixedBacklog'); - -const chai = require('chai'); -chai.should(); -const sinon = require('sinon'); - -describe('fixedBacklog controller implementation', () => { - - let controller; - - describe('#init', () => { - - let msg = {totalClients: 1}; - let opts = { - transaction_load: 30, - startingTps: 10 - }; - - it('should set the sleep time for a single client if no clients are specified and the startingTps is specified', () => { - let msg = {}; - controller = new FixedBacklog.createRateController(opts); - controller.init(msg); - controller.sleepTime.should.equal(100); - }); - - it('should set the sleep time for a single client if no clients are specified and the startingTps is not specified', () => { - let msg = {}; - let opts = { - transaction_load: 90 - }; - controller = new FixedBacklog.createRateController(opts); - controller.init(msg); - controller.sleepTime.should.equal(1000); - }); - - it('should set the sleep time for a multiple clients it the startingTps is specified', () => { - controller = new FixedBacklog.createRateController(opts); - controller.init(msg); - controller.sleepTime.should.equal(100); - }); - - it('should set the sleep time for a multiple clients it the startingTps is not specified', () => { - let opts = { - transaction_load: 30 - }; - controller = new FixedBacklog.createRateController(opts); - controller.init(msg); - controller.sleepTime.should.equal(1000); - }); - - it('should set a default transaction backlog for multiple clients if not specified', () => { - controller = new FixedBacklog.createRateController({}); - controller.init(msg); - controller.unfinished_per_worker.should.equal(10); - }); - - it('should set the transaction backlog for multiple clients if specified', () => { - controller = new FixedBacklog.createRateController(opts); - controller.init(msg); - controller.unfinished_per_worker.should.equal(30); - }); - - }); - - describe('#applyRateControl', () => { - - let sleepStub; - let opts = { - transaction_load: 30, - startingTps: 10 - }; - - beforeEach(() => { - sleepStub = sinon.stub(); - FixedBacklog.__set__('Sleep', sleepStub); - - controller = new FixedBacklog.createRateController(opts); - controller.sleepTime = 1000; - controller.unfinished_per_worker = 30; - }); - - it('should sleep if resultStats.length < 2', async () => { - await controller.applyRateControl(null, 1, [], []); - sinon.assert.calledOnce(sleepStub); - sinon.assert.calledWith(sleepStub, 1000); - }); - - it ('should sleep if no successful results are available', async () => { - await controller.applyRateControl(null, 1, [], [{}]); - sinon.assert.calledOnce(sleepStub); - sinon.assert.calledWith(sleepStub, 1000); - }); - - it ('should sleep if no delay results are available', async () => { - await controller.applyRateControl(null, 1, [], [{}]); - sinon.assert.calledOnce(sleepStub); - sinon.assert.calledWith(sleepStub, 1000); - }); - - it ('should not sleep if backlog transaction is below target', async () => { - let idx = 50; - let currentResults = []; - let item = { - succ: 5, - length: 30, - delay: { - sum: 5 - } - }; - const resultStats = []; - resultStats.push(item); - resultStats.push(item); - - await controller.applyRateControl(null, idx, currentResults, resultStats); - sinon.assert.notCalled(sleepStub); - }); - - it ('should sleep if backlog transaction is at or above target', async () => { - let idx = 50; - let currentResults = []; - let item = { - succ: 5, - length: 5, - delay: { - sum: 5 - } - }; - const resultStats = []; - resultStats.push(item); - resultStats.push(item); - - await controller.applyRateControl(null, idx, currentResults, resultStats); - - sinon.assert.calledOnce(sleepStub); - }); - - it ('should sleep for a count of the load error and the current average delay', async () => { - let idx = 50; - let currentResults = []; - let item = { - succ: 9, - fail: 1, - length: 10, - delay: { - sum: 5 - }, - create: { - min: 1 - }, - final: { - last: 21 - } - }; - const resultStats = []; - resultStats.push(item); - resultStats.push(item); - - await controller.applyRateControl(null, idx, currentResults, resultStats); - - const completeTransactions = resultStats[0].length - currentResults.length; - const unfinished = idx - completeTransactions; - - const backlogDifference = unfinished - 30; - const determinedTPS = 0.5; - - sinon.assert.calledOnce(sleepStub); - sinon.assert.calledWith(sleepStub, backlogDifference*(1000/determinedTPS)); - }); - - it('should log the backlog error as a debug message', async () => { - - const FakeLogger = { - debug : () => {}, - error: () => {} - }; - - let debugStub = sinon.stub(FakeLogger, 'debug'); - FixedBacklog.__set__('Logger', FakeLogger); - - let idx = 50; - let currentResults = []; - let item = { - succ: 5, - length: 5, - delay: { - sum: 5 - } - }; - const resultStats = []; - resultStats.push(item); - resultStats.push(item); - - await controller.applyRateControl(null, idx, currentResults, resultStats); - - const completeTransactions = resultStats[0].length - currentResults.length; - const unfinshed = idx - completeTransactions; - - const error = unfinshed - 30; - const message = 'Difference between current and desired transaction backlog: ' + error; - - sinon.assert.calledOnce(debugStub); - sinon.assert.calledWith(debugStub, message); - - }); - - }); - -}); diff --git a/packages/caliper-core/test/worker/rate-control/fixedFeedbackRate.js b/packages/caliper-core/test/worker/rate-control/fixedFeedbackRate.js index d7dae45ef1..2ade6da208 100644 --- a/packages/caliper-core/test/worker/rate-control/fixedFeedbackRate.js +++ b/packages/caliper-core/test/worker/rate-control/fixedFeedbackRate.js @@ -16,6 +16,8 @@ const rewire = require('rewire'); const FixedFeedbackRate = rewire('../../../lib/worker/rate-control/fixedFeedbackRate'); +const TestMessage = require('../../../lib/common/messages/testMessage'); +const TransactionStatisticsCollector = require('../../../lib/common/core/transaction-statistics-collector'); const chai = require('chai'); chai.should(); @@ -23,227 +25,124 @@ const sinon = require('sinon'); describe('fixedFeedbackRate controller implementation', () => { - describe('#init', () => { + describe('#constructor', () => { let controller; - let opts = { - tps: 100, - maximum_transaction_load: 100 - }; - - beforeEach(() => { - controller = new FixedFeedbackRate.createRateController(opts); - }); - - it('should set the sleepTime for a single client if no clients are specified', () => { - let msg = {}; - controller.init(msg); - - // if the tps per client is is 100, then the sleep time is (1000/100) = 10 - controller.sleepTime.should.equal(10); - }); - - it('should set the sleepTime for a single client to 0 if no clients are specified and tps is 0', () => { - let msg = {}; - let opts = { - tps: 0, - maximum_transaction_load: 100 + let testMessage; + beforeEach( () => { + const msgContent = { + label: 'query2', + rateControl: { + type: 'fixed-feedback-rate', + opts: {} + }, + workload: { + module:'./../queryByChannel.js' + }, + testRound:0, + txDuration:250, + totalClients:2 }; - controller = new FixedFeedbackRate.createRateController(opts); - - controller.init(msg); - controller.sleepTime.should.equal(0); + testMessage = new TestMessage('test', [], msgContent); }); - it('should set the sleepTime for multiple clients', () => { - let msg = {totalClients: 4}; - controller.init(msg); - - // if the number of clients is 4, then the tps per client is (100/4) = 25 and the sleeptime is (1000/25) = 40 - controller.sleepTime.should.equal(40); + it('should set the sleepTime for a single client if no options are specified', () => { + testMessage.content.totalClients = 1; + controller = new FixedFeedbackRate.createRateController(testMessage, {}, 0); + controller.generalSleepTime.should.equal(100); }); - it('should set the sleepTime to zero if 0 tps is specified', () => { - controller = new FixedFeedbackRate.createRateController({tps: 0}); - let msg = {totalClients: 1}; - controller.init(msg); - controller.sleepTime.should.equal(0); + it('should set unfinishedPerWorker for multiple workers if specified', () => { + testMessage.content.rateControl.opts = { transactionLoad: 50 }; + controller = new FixedFeedbackRate.createRateController(testMessage, {}, 0); + controller.unfinishedPerWorker.should.equal(25); }); - it ('should set the sleep_time if specified', () => { - controller = new FixedFeedbackRate.createRateController({sleep_time: 50}); - let msg = {totalClients: 1}; - controller.init(msg); - controller.sleep_time.should.equal(50); + it('should set unfinishedPerWorker for multiple workers if not specified', () => { + testMessage.content.rateControl.opts = { }; + controller = new FixedFeedbackRate.createRateController(testMessage, {}, 0); + controller.unfinishedPerWorker.should.equal(5); }); - it ('should set a default sleep_time if not specified', () => { - let msg = {totalClients: 1}; - controller.init(msg); - controller.sleep_time.should.equal(100); - }); - - it ('should set the transaction backlog for multiple workers if specified', () => { - let msg = {totalClients: 2}; - controller.init(msg); - controller.unfinished_per_worker.should.equal(50); - }); - - it ('should set a default transaction backlog for multiple workers if not specified', () => { - controller = new FixedFeedbackRate.createRateController({}); - let msg = {totalClients: 1}; - controller.init(msg); - controller.unfinished_per_worker.should.equal(100); - }); - - it ('should set zero_succ_count to 0', () => { - let msg = {totalClients: 1}; - controller.init(msg); - controller.zero_succ_count.should.equal(0); + it('should set zeroSuccessfulCounter to 0', () => { + testMessage.content.rateControl.opts = { }; + controller = new FixedFeedbackRate.createRateController(testMessage, {}, 0); + controller.zeroSuccessfulCounter.should.equal(0); }); it ('should set the total sleep time to 0', () => { - let msg = {totalClients: 1}; - controller.init(msg); - controller.total_sleep_time.should.equal(0); + testMessage.content.rateControl.opts = { }; + controller = new FixedFeedbackRate.createRateController(testMessage, {}, 0); + controller.totalSleepTime.should.equal(0); }); }); describe('#applyRateController', () => { - let controller, sleepStub, clock; - - let opts = { - tps: 100, - maximum_transaction_load: 100 - }; + let controller, sleepStub, txnStats, clock; + + beforeEach( () => { + const msgContent = { + label: 'query2', + rateControl: { + type: 'fixed-feedback-rate', + opts: {} + }, + workload: { + module:'./../queryByChannel.js' + }, + testRound:0, + txDuration:250, + totalClients:2 + }; - beforeEach(() => { clock = sinon.useFakeTimers(); sleepStub = sinon.stub(); FixedFeedbackRate.__set__('util.sleep', sleepStub); - controller = new FixedFeedbackRate.createRateController(opts); - controller.maximum_transaction_load = 100; - controller.sleepTime = 50; - controller.sleep_time = 100; - controller.zero_succ_count = 0; + const testMessage = new TestMessage('test', [], msgContent); + txnStats = new TransactionStatisticsCollector(); + controller = new FixedFeedbackRate.createRateController(testMessage, txnStats, 0); }); - it ('should not sleep if the sleepTime is 0', () => { - let start = 0; - let idx = 100; - let resultStats = [ - { - succ: 15, - fail: 5 - } - ]; - controller.sleepTime = 0; - controller.applyRateControl(start, idx, [], resultStats); - - // should have not called the sleep method - sinon.assert.notCalled(sleepStub); + afterEach(() => { + clock.restore(); }); - it ('should not sleep if id < maximum_transaction_load', () => { - let start = 0; - let idx = 50; - let resultStats = [ - { - succ: 15, - fail: 5 - } - ]; - controller.applyRateControl(start, idx, [], resultStats); + it('should not sleep if the generalSleepTime is 0', () => { + controller.generalSleepTime = 0; + txnStats.stats.txCounters.totalSubmitted = 2000; + controller.applyRateControl(); // should have not called the sleep method sinon.assert.notCalled(sleepStub); }); - it ('should sleep if the elapsed time difference is greater than 5ms', () => { - let start = 0; - let idx = 100; - controller.sleepTime = 10; - controller.total_sleep_time = 50; - clock.tick(100); - - let diff = (10 * idx - ((Date.now() - 50) - start)); - - controller.applyRateControl(start, idx, null, []); - - // should have called the sleep method with a value equal to diff - sinon.assert.calledOnce(sleepStub); - sinon.assert.calledWith(sleepStub, diff); - }); - - it ('should not sleep if there are no aggregated stats from the previous round', () => { - let start = 0; - let idx = 100; - let resultStats = []; - controller.applyRateControl(start, idx, [], resultStats); + it('should not sleep if there are no unfinished transactions', () => { + txnStats.stats.txCounters.totalSubmitted = 0; + controller.applyRateControl(); // should have not called the sleep method sinon.assert.notCalled(sleepStub); }); - it ('should not sleep if backlog transaction is below half the target', () => { - let start = 50; - let idx = 100; - let resultStats = [ - { - succ: 50, - fail: 4 - } - ]; - controller.applyRateControl(start, idx, [], resultStats); + it('should not sleep if backlog transaction is below half the target', () => { + txnStats.stats.txCounters.totalSubmitted = 1000; + txnStats.stats.txCounters.totalFinished = 999; + controller.generalSleepTime = 1; + controller.applyRateControl(); // should have not called the sleep method sinon.assert.notCalled(sleepStub); }); - it ('should determine the sleeptime for waiting until successful transactions occur', () => { - let start = 0; - let idx = 2; - let resultStats = [ - { - succ: 1, fail: 0 - }, - { - succ: 0, fail: 1 - } - ]; - controller.maximum_transaction_load = 2; - controller.sleepTime = 1; - controller.total_sleep_time = 2; - - controller.applyRateControl(start, idx, [], resultStats); - - // should have called the sleep method with a value equal to sleep_time - sinon.assert.calledOnce(sleepStub); - sinon.assert.calledWith(sleepStub, controller.sleep_time); - }); - - it ('should determines the sleep time according to the current number of unfinished transactions with the configure one', () => { - let start = 0; - let idx = 2; - let resultStats = [ - { - succ: 0, fail: 0 - }, - { - succ: 2, fail: 0 - } - ]; - - controller.unfinished_per_worker = 2; - controller.sleepTime = 1; - controller.total_sleep_time = 2; - - controller.applyRateControl(start, idx, [], resultStats); + it ('should sleep if the elapsed time difference is greater than 5ms', () => { + txnStats.stats.txCounters.totalSubmitted = 100; + txnStats.stats.txCounters.totalFinished = 2; + controller.applyRateControl(); - // should have called the sleep method with a value equal to sleep_time + // should have called the sleep method with a value equal to diff sinon.assert.calledOnce(sleepStub); - sinon.assert.calledWith(sleepStub, controller.sleep_time); + sinon.assert.calledWith(sleepStub, 20000); }); }); }); diff --git a/packages/caliper-core/test/worker/rate-control/fixedLoad.js b/packages/caliper-core/test/worker/rate-control/fixedLoad.js new file mode 100644 index 0000000000..b16ac745ad --- /dev/null +++ b/packages/caliper-core/test/worker/rate-control/fixedLoad.js @@ -0,0 +1,183 @@ +/* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +'use strict'; + +const rewire = require('rewire'); +const FixedLoad = rewire('../../../lib/worker/rate-control/fixedLoad'); +const TestMessage = require('../../../lib/common/messages/testMessage'); +const TransactionStatisticsCollector = require('../../../lib/common/core/transaction-statistics-collector'); + +const chai = require('chai'); +chai.should(); +const sinon = require('sinon'); + +describe('fixedLoad controller implementation', () => { + + describe('#constructor', () => { + + let controller; + let testMessage; + beforeEach( () => { + const msgContent = { + label: 'query2', + rateControl: { + type: 'fixed-load', + opts: { + startTps:10, + transactionLoad:20 + } + }, + workload: { + module:'./../queryByChannel.js' + }, + testRound:0, + txDuration:250, + totalClients:2 + }; + testMessage = new TestMessage('test', [], msgContent); + }); + + it('should set the sleep time for a single client if a single worker is specified and the startingTps is not specified', () => { + testMessage.content.rateControl.opts = {}; + testMessage.content.totalClients = 1; + controller = new FixedLoad.createRateController(testMessage, {}, 0); + controller.sleepTime.should.equal(200); + }); + + it('should set the sleep time for a single client if no clients are specified and the startingTps is specified', () => { + testMessage.content.rateControl.opts = { startTps: 50 }; + testMessage.content.totalClients = 1; + controller = new FixedLoad.createRateController(testMessage, {}, 0); + controller.sleepTime.should.equal(20); + }); + + it('should set the sleep time for a multiple workers it the startingTps is not specified', () => { + testMessage.content.rateControl.opts = {}; + testMessage.content.totalClients = 2; + controller = new FixedLoad.createRateController(testMessage, {}, 0); + controller.sleepTime.should.equal(400); + }); + + it('should set the sleep time for a multiple workers if the startingTps is specified', () => { + testMessage.content.rateControl.opts = { startTps: 50 }; + testMessage.content.totalClients = 2; + controller = new FixedLoad.createRateController(testMessage, {}, 0); + controller.sleepTime.should.equal(40); + }); + + it('should set a default transaction backlog for multiple clients if not specified', () => { + testMessage.content.rateControl.opts = { startTps: 50 }; + controller = new FixedLoad.createRateController(testMessage, {}, 0); + controller.targetLoad.should.equal(5); + }); + + it('should set the transaction backlog for multiple clients if specified', () => { + controller = new FixedLoad.createRateController(testMessage, {}, 0); + controller.targetLoad.should.equal(10); + }); + + }); + + describe('#applyRateControl', () => { + + let sleepStub; + let txnStats; + let controller; + + beforeEach(() => { + sleepStub = sinon.stub(); + FixedLoad.__set__('Sleep', sleepStub); + + const msgContent = { + label: 'query2', + rateControl: { + type: 'fixed-load', + opts: { + startTps:10, + transactionLoad:20 + } + }, + workload: { + module:'./../queryByChannel.js' + }, + testRound:0, + txDuration:250, + totalClients:2 + }; + const testMessage = new TestMessage('test', [], msgContent); + txnStats = new TransactionStatisticsCollector(); + controller = new FixedLoad.createRateController(testMessage, txnStats, 0); + }); + + it ('should sleep if no successful results are available', async () => { + await controller.applyRateControl(); + sinon.assert.calledOnce(sleepStub); + sinon.assert.calledWith(sleepStub, 200); + }); + + it ('should not sleep if backlog transaction is below target', async () => { + txnStats.stats.txCounters.totalSubmitted = 20; + txnStats.stats.txCounters.totalFinished = 20; + + await controller.applyRateControl(); + sinon.assert.notCalled(sleepStub); + }); + + it ('should sleep if backlog transaction is at or above target', async () => { + txnStats.stats.txCounters.totalSubmitted = 20; + txnStats.stats.txCounters.totalFinished = 0; + + await controller.applyRateControl(); + sinon.assert.calledOnce(sleepStub); + }); + + it ('should sleep for a count of the load error and the current average delay', async () => { + txnStats.stats.txCounters.totalSubmitted = 80; + txnStats.stats.txCounters.totalFinished = 40; + + txnStats.stats.txCounters.totalSuccessful = 40; + txnStats.stats.latency.successful.total = 10000; + + await controller.applyRateControl(); + sinon.assert.calledOnce(sleepStub); + sinon.assert.calledWith(sleepStub, 7500); + }); + + it('should log the backlog error as a debug message', async () => { + + const FakeLogger = { + debug : () => {}, + error: () => {} + }; + + let debugStub = sinon.stub(FakeLogger, 'debug'); + FixedLoad.__set__('Logger', FakeLogger); + + txnStats.stats.txCounters.totalSubmitted = 80; + txnStats.stats.txCounters.totalFinished = 40; + + txnStats.stats.txCounters.totalSuccessful = 40; + txnStats.stats.latency.successful.total = 10000; + + await controller.applyRateControl(); + const message = 'Difference between current and desired transaction backlog: 30'; + sinon.assert.calledOnce(debugStub); + sinon.assert.calledWith(debugStub, message); + + }); + + }); + +}); diff --git a/packages/caliper-core/test/worker/rate-control/fixedRate.js b/packages/caliper-core/test/worker/rate-control/fixedRate.js index a9a09b45eb..3752daeb43 100644 --- a/packages/caliper-core/test/worker/rate-control/fixedRate.js +++ b/packages/caliper-core/test/worker/rate-control/fixedRate.js @@ -16,6 +16,8 @@ const rewire = require('rewire'); const FixedRate = rewire('../../../lib/worker/rate-control/fixedRate'); +const TestMessage = require('../../../lib/common/messages/testMessage'); +const TransactionStatisticsCollector = require('../../../lib/common/core/transaction-statistics-collector'); const chai = require('chai'); chai.should(); @@ -23,87 +25,93 @@ const sinon = require('sinon'); describe('fixedRate controller implementation', () => { - let controller; - let opts = {tps: 40}; - - describe('#init', () => { - - beforeEach(() => { - controller = new FixedRate.createRateController(opts); + describe('#constructor', () => { + + let controller; + let testMessage; + beforeEach( () => { + const msgContent = { + label: 'query2', + rateControl: { + type: 'fixed-rate', + opts: {} + }, + workload: { + module:'./../queryByChannel.js' + }, + testRound:0, + txDuration:250, + totalClients:2 + }; + testMessage = new TestMessage('test', [], msgContent); }); - it('should set the sleep time for a single client if no clients are specified', () => { - let msg = {}; - controller.init(msg); - controller.sleepTime.should.equal(25); + it('should set a default sleep time if no options passed', () => { + testMessage.content.totalClients = 1; + controller = new FixedRate.createRateController(testMessage, {}, 0); + controller.sleepTime.should.equal(100); }); it('should set the sleep time for a single client', () => { - let msg = {totalClients: 1}; - controller.init(msg); - controller.sleepTime.should.equal(25); + testMessage.content.totalClients = 1; + testMessage.content.rateControl.opts = { tps: 50 }; + controller = new FixedRate.createRateController(testMessage, {}, 0); + controller.sleepTime.should.equal(20); }); it('should set the sleep time for multiple clients', () => { - let msg = {totalClients: 4}; - controller.init(msg); - controller.sleepTime.should.equal(100); - }); - - it('should set the sleep time to zero if 0 tps specified', () => { - controller = new FixedRate.createRateController({tps: 0}); - let msg = {totalClients: 1}; - controller.init(msg); - controller.sleepTime.should.equal(0); + testMessage.content.totalClients = 2; + testMessage.content.rateControl.opts = { tps: 50 }; + controller = new FixedRate.createRateController(testMessage, {}, 0); + controller.sleepTime.should.equal(40); }); }); describe('#applyRateControl', () => { - let sleepStub; - let clock; + let controller, sleepStub, txnStats, clock; beforeEach(() => { + const msgContent = { + label: 'query2', + rateControl: { + type: 'fixed-feedback-rate', + opts: {} + }, + workload: { + module:'./../queryByChannel.js' + }, + testRound:0, + txDuration:250, + totalClients:2 + }; + clock = sinon.useFakeTimers(); sleepStub = sinon.stub(); FixedRate.__set__('Sleep', sleepStub); - controller = new FixedRate.createRateController(opts); - controller.sleepTime = 10000; + const testMessage = new TestMessage('test', [], msgContent); + txnStats = new TransactionStatisticsCollector(); + controller = new FixedRate.createRateController(testMessage, txnStats, 0); }); afterEach(() => { clock.restore(); }); - it('should sleep for the full ammount of time if there is zero elapsed time', () => { - controller.applyRateControl(Date.now(), 1, []); - sinon.assert.calledOnce(sleepStub); - sinon.assert.calledWith(sleepStub, 10000); - }); - - it('should reduce the sleep time based on the elapsed time difference', () => { - let startTime = Date.now(); - clock.tick(5000); - controller.applyRateControl(startTime, 1, []); - sinon.assert.calledOnce(sleepStub); - sinon.assert.calledWith(sleepStub, 5000); - }); - - it('should not sleep if the elapsed time difference is below the 5ms threshold', () => { - let startTime = Date.now(); - clock.tick(99996); - controller.applyRateControl(startTime, 1, []); - sinon.assert.notCalled(sleepStub); - }); - it('should not sleep if the sleepTime is zero', () => { controller.sleepTime = 0; - controller.applyRateControl(Date.now(), 1, []); + controller.applyRateControl(); sinon.assert.notCalled(sleepStub); }); + it('should sleep based on the difference between the required increment time, and the elapsed time', () => { + txnStats.stats.txCounters.totalSubmitted = 100; + controller.applyRateControl(); + sinon.assert.calledOnce(sleepStub); + sinon.assert.calledWith(sleepStub, 20000); + }); }); }); diff --git a/packages/caliper-core/test/worker/rate-control/linearRate.js b/packages/caliper-core/test/worker/rate-control/linearRate.js index 4b6989467b..92aa0ff0f0 100644 --- a/packages/caliper-core/test/worker/rate-control/linearRate.js +++ b/packages/caliper-core/test/worker/rate-control/linearRate.js @@ -15,7 +15,10 @@ 'use strict'; const rewire = require('rewire'); +const LinearRateRewire = rewire('../../../lib/worker/rate-control/linearRate'); const LinearRate = rewire('../../../lib/worker/rate-control/linearRate'); +const TestMessage = require('../../../lib/common/messages/testMessage'); +const TransactionStatisticsCollector = require('../../../lib/common/core/transaction-statistics-collector'); const chai = require('chai'); chai.should(); @@ -23,162 +26,210 @@ const sinon = require('sinon'); describe('linearRate controller implementation', () => { - let controller; - let opts = { - startingTps: 20, - finishingTps: 80 - }; - describe('#_interpolateFromIndex', () => { + + let controller, testMessage, txnStats; + beforeEach( () => { + const msgContent = { + label: 'query2', + rateControl: { + type: 'linear-rate', + opts: {} + }, + workload: { + module:'./../queryByChannel.js' + }, + testRound:0, + txDuration:250, + totalClients:2 + }; + testMessage = new TestMessage('test', [], msgContent); + }); + it('should return value interpolated from index', () => { - controller = new LinearRate.createRateController(); + txnStats = new TransactionStatisticsCollector(); + txnStats.stats.txCounters.totalSubmitted = 5; + controller = new LinearRate.createRateController(testMessage, txnStats, 0); controller.startingSleepTime = 3; controller.gradient = 2; - const idx = 5; // If the starting sleeptime is 3ms, the gradient is 2 and the index is 5, the returned interpolated value should be ((3 + (5*2)) = 13 - const value = controller._interpolateFromIndex(null, idx); - value.should.equal(13); + controller._interpolateFromIndex().should.equal(13); }); }); describe('#_interpolateFromTime', () => { - let clock; + let clock, controller, testMessage, txnStats; - it('should return value interpolated from time', () => { + beforeEach( () => { clock = sinon.useFakeTimers(); - controller = new LinearRate.createRateController(opts); + const msgContent = { + label: 'query2', + rateControl: { + type: 'linear-rate', + opts: {} + }, + workload: { + module:'./../queryByChannel.js' + }, + testRound:0, + txDuration:250, + totalClients:2 + }; + testMessage = new TestMessage('test', [], msgContent); + }); + + afterEach(() => { + clock.restore(); + }); + + it('should return value interpolate from time', () => { + txnStats = new TransactionStatisticsCollector(); + txnStats.stats.metadata.roundStartTime = 5; + controller = new LinearRate.createRateController(testMessage, txnStats, 0); controller.startingSleepTime = 3; controller.gradient = 2; - const start = 5; clock.tick(5); // If the starting sleeptime is 3ms, the gradient is 2 and start is 5ms, the returned interpolated value should be ((3 + (5-5)*2)) = 3 - const value = controller._interpolateFromTime(start, null); - value.should.equal(3); - - clock.restore(); + controller._interpolateFromTime().should.equal(3); }); }); - describe('#init', () => { - let clock; + describe('#constructor', () => { + let clock, controller, testMessage; - beforeEach(() => { + beforeEach( () => { clock = sinon.useFakeTimers(); - controller = new LinearRate.createRateController(opts); + const msgContent = { + label: 'query2', + rateControl: { + type: 'linear-rate', + opts: {} + }, + workload: { + module:'./../queryByChannel.js' + }, + testRound:0, + txDuration:250, + totalClients:2 + }; + testMessage = new TestMessage('test', [], msgContent); }); afterEach(() => { clock.restore(); }); - it('should set the starting sleep time based on starting tps and total number of clients', () => { - let msg = {totalClients: 6}; - controller.init(msg); + testMessage.content.totalClients = 6; + testMessage.content.rateControl.opts = { startingTps: 20 }; + controller = new LinearRate.createRateController(testMessage, {}, 0); // If there are 6 clients with an initial 20 TPS goal, the starting sleep time should be (1000/(20/6)) = 300ms controller.startingSleepTime.should.equal(300); }); - it('should set the gradient based on linear interpolation between two points with index and sleep time axis', () => { - let msg = { - totalClients: 6, - numb: 5 - }; - controller.init(msg); + it('should set the gradient based on linear interpolation between two points separated based on a txn count', () => { + testMessage.content.totalClients = 6; + testMessage.content.rateControl.opts = { startingTps: 20, finishingTps: 80 }; + testMessage.content.numb = 5; + controller = new LinearRate.createRateController(testMessage, {}, 0); // if there are 6 clients with a starting sleep time of (1000/(20/6) = 300, a finishing sleep time of (1000/(80/6)) = 75, and a duration of 5, // the gradient should be ((75 - 300) / 5) = -45 controller.gradient.should.equal(-45); }); - it('should set the gradient based on linear interpolation between two points with time and sleep time axis', () => { - let msg = { - totalClients: 6, - txDuration: 5, - }; - controller.init(msg); + it('should set the gradient based on linear interpolation between two points separated based on a duration count', () => { + testMessage.content.totalClients = 6; + testMessage.content.rateControl.opts = { startingTps: 20, finishingTps: 80 }; + testMessage.content.txDuration = 5; + controller = new LinearRate.createRateController(testMessage, {}, 0); - // If there are 6 clients with a starting sleep time of (1000/(20/6) = 300, a finishing sleep time of (1000/(80/6)) = 75, and a duration of (5*1000) = 5000, + // If there are 6 clients with a starting sleep time of (1000/(20/6) = 300, a finishing sleep time of (1000/(80/6)) = 75, and a duration of (5*1000) = 5000, // the gradient should be ((75 - 300) / 5000) = -0.045 controller.gradient.should.equal(-0.045); }); - it('should determine the interpolated value when the number of transactions generated in the round is specified', () => { - let msg = { - totalClients: 6, - numb: 5 - }; - const idx = 5; - controller.init(msg); + it('should assign _interpolateFromIndex to _interpolate if txCount based', () => { + testMessage.content.totalClients = 6; + testMessage.content.rateControl.opts = { startingTps: 20, finishingTps: 80 }; + testMessage.content.numb = 5; + + const txnStats = new TransactionStatisticsCollector(); + const mySpy = sinon.spy(txnStats, 'getTotalSubmittedTx'); + controller = new LinearRate.createRateController(testMessage, txnStats, 0); - // if there are 5 transactions to be generated in the round with an index of 5, a starting sleep time of (1000/(20/6)) = 300ms and a gradient of (((1000/(80/6)) - 300) / 5) = -45 - // then the value interpolated (300 + 5*-45) = 75 - controller._interpolateFromIndex(null, idx).should.equal(75); + controller._interpolate(); + sinon.assert.called(mySpy); }); - it('should determine the interpolated value when the number of transactions generated in the round is not specified', () => { - let msg = { - totalClients: 6, - txDuration: 5 - }; - const start = 5; - clock.tick(5); - controller.init(msg); + it('should assign _interpolateFromTime to _interpolate if txCount based', () => { + testMessage.content.totalClients = 6; + testMessage.content.rateControl.opts = { startingTps: 20, finishingTps: 80 }; + testMessage.content.txDuration = 5; + + const txnStats = new TransactionStatisticsCollector(); + const mySpy = sinon.spy(txnStats, 'getRoundStartTime'); + controller = new LinearRate.createRateController(testMessage, txnStats, 0); - // if the number of transaction generated in the round is 5, the start is 5, the starting sleep time is (1000/(20/6)) = 300ms and the gradient is (((1000/(80/6)) - 300) / 5) = -0.045, - // then the value interpolated (300 + (5-5)*-0.045) = 300 - controller._interpolateFromTime(start, null).should.equal(300); + controller._interpolate(); + sinon.assert.called(mySpy); }); }); describe('#applyRateController', () => { - let sleepStub; + let controller, sleepStub, txnStats, clock; beforeEach(() => { + const msgContent = { + label: 'query2', + rateControl: { + type: 'linear-feedback-rate', + opts: { startingTps: 20, finishingTps: 80 } + }, + workload: { + module:'./../queryByChannel.js' + }, + testRound:0, + txDuration:250, + totalClients:2 + }; + + clock = sinon.useFakeTimers(); + sleepStub = sinon.stub(); - LinearRate.__set__('util.sleep', sleepStub); + LinearRateRewire.__set__('Sleep', sleepStub); - controller = new LinearRate.createRateController(opts); + const testMessage = new TestMessage('test', [], msgContent); + txnStats = new TransactionStatisticsCollector(); + controller = new LinearRateRewire.createRateController(testMessage, txnStats, 0); }); - - it('should sleep for a duration of current sleep time if greater than 5ms', () => { - const currentSleepTime = 6; - let interpolateStub = sinon.stub().returns(currentSleepTime); - controller._interpolateFromIndex = interpolateStub; - let msg = { - totalClients: 6, - numb: 5 - }; - controller.init(msg); - const idx = 5; + afterEach(() => { + clock.restore(); + }); - controller.applyRateControl(null, idx, null, null); + it('should sleep for a duration of current sleep time if greater than 5ms', () => { + txnStats.stats.txCounters.totalSubmitted = 1000; + txnStats.stats.metadata.roundStartTime = 0; + controller.startingSleepTime = 50; + controller.applyRateControl(); // should have called the sleep method with current sleep time of 6ms sinon.assert.calledOnce(sleepStub); - sinon.assert.calledWith(sleepStub, currentSleepTime); + sinon.assert.calledWith(sleepStub, 50); }); it('should do nothing where current sleep time is less than or equal to 5ms', () => { - let currentSleepTime = 4; - let interpolateStub = sinon.stub().returns(currentSleepTime); - controller._interpolateFromTime = interpolateStub; - - let msg = { - totalClients: 6, - txDuration: 5 - }; - controller.init(msg); - const start = 5; - - controller.applyRateControl(start, null, null, null); + txnStats.stats.txCounters.totalSubmitted = 0; + txnStats.stats.metadata.roundStartTime = 0; + controller.startingSleepTime = 0; + controller.applyRateControl(); // should not have called the sleep method sinon.assert.notCalled(sleepStub); diff --git a/packages/caliper-core/test/worker/rate-control/maxRate.js b/packages/caliper-core/test/worker/rate-control/maxRate.js index 5e2a42ee0e..918e5a40d5 100644 --- a/packages/caliper-core/test/worker/rate-control/maxRate.js +++ b/packages/caliper-core/test/worker/rate-control/maxRate.js @@ -16,6 +16,8 @@ const rewire = require('rewire'); const MaxRate = rewire('../../../lib/worker/rate-control/maxRate'); +const TestMessage = require('../../../lib/common/messages/testMessage'); +const TransactionStatisticsCollector = require('../../../lib/common/core/transaction-statistics-collector'); const chai = require('chai'); chai.should(); @@ -23,401 +25,367 @@ const sinon = require('sinon'); describe('maxRate controller implementation', () => { - let sandbox; + describe('#constructor', () => { - beforeEach(() => { - sandbox = sinon.createSandbox(); - }); + let sandbox, controller, testMessage; + beforeEach(() => { + sandbox = sinon.createSandbox(); - afterEach( () => { - sandbox.restore(); - }); + const msgContent = { + label: 'query2', + rateControl: { + type: 'maximum-rate', + opts: {} + }, + workload: { + module:'./../queryByChannel.js' + }, + testRound:0, + txDuration:250, + totalClients:1 + }; + testMessage = new TestMessage('test', [], msgContent); + }); + + afterEach( () => { + sandbox.restore(); + }); - describe('#init', () => { it('should set a default starting TPS for single or multiple workers', () => { - let opts = {}; - let msg = {}; - let controller = new MaxRate.createRateController(opts); - controller.init(msg); + controller = new MaxRate.createRateController(testMessage, {}, 0); controller.tpsSettings.current.should.equal(5); - msg.totalClients = 2; - controller.init(msg); + testMessage.content.totalClients = 2; + controller = new MaxRate.createRateController(testMessage, {}, 0); controller.tpsSettings.current.should.equal(2.5); }); it('should set a starting TPS for single or multiple workers', () => { - let opts = { + testMessage.content.rateControl.opts = { tps: 10 }; - let msg = {}; - let controller = new MaxRate.createRateController(opts); - controller.init(msg); + controller = new MaxRate.createRateController(testMessage, {}, 0); controller.tpsSettings.current.should.equal(10); - msg.totalClients = 2; - controller.init(msg); + testMessage.content.totalClients = 2; + controller = new MaxRate.createRateController(testMessage, {}, 0); controller.tpsSettings.current.should.equal(5); }); it('should set a default step size for single or multiple workers', () => { - let opts = {}; - let msg = {}; - let controller = new MaxRate.createRateController(opts); - controller.init(msg); + controller = new MaxRate.createRateController(testMessage, {}, 0); controller.step.should.equal(5); - msg.totalClients = 2; - controller.init(msg); + testMessage.content.totalClients = 2; + controller = new MaxRate.createRateController(testMessage, {}, 0); controller.step.should.equal(2.5); }); it('should set a specified step size for single or multiple workers', () => { - let opts = { + testMessage.content.rateControl.opts = { step: 10 }; - let msg = {}; - let controller = new MaxRate.createRateController(opts); - controller.init(msg); + controller = new MaxRate.createRateController(testMessage, {}, 0); controller.step.should.equal(10); - msg.totalClients = 2; - controller.init(msg); + testMessage.content.totalClients = 2; + controller = new MaxRate.createRateController(testMessage, {}, 0); controller.step.should.equal(5); }); it('should set a default sample interval for single or multiple workers', () => { - let opts = {}; - let msg = {}; - let controller = new MaxRate.createRateController(opts); - controller.init(msg); - controller.sampleInterval.should.equal(10); - - msg.totalClients = 2; - controller.init(msg); - controller.sampleInterval.should.equal(10); + controller = new MaxRate.createRateController(testMessage, {}, 0); + controller.sampleInterval.should.equal(10000); + + testMessage.content.totalClients = 2; + controller = new MaxRate.createRateController(testMessage, {}, 0); + controller.sampleInterval.should.equal(10000); }); it('should set a sample interval if specified for single or multiple workers', () => { - let opts = { + testMessage.content.rateControl.opts = { sampleInterval: 20 }; - let msg = {}; - let controller = new MaxRate.createRateController(opts); - controller.init(msg); - controller.sampleInterval.should.equal(20); - - msg.totalClients = 2; - controller.init(msg); - controller.sampleInterval.should.equal(20); + controller = new MaxRate.createRateController(testMessage, {}, 0); + controller.sampleInterval.should.equal(20000); + + testMessage.content.totalClients = 2; + controller = new MaxRate.createRateController(testMessage, {}, 0); + controller.sampleInterval.should.equal(20000); }); }); describe('#applyRateControl', async () => { + let sandbox; let sleepStub; let controller; - let opts = {}; + let txnStats; + let clock; beforeEach(() => { - controller = new MaxRate.createRateController(opts); - sleepStub = sandbox.stub(controller, 'applySleepInterval'); - }); - it('should sleep if resultStats.length < 2',async () => { - let updateSpy = sandbox.spy(controller, 'updateOccurred'); - await controller.applyRateControl(null, 1, [], [{}]); + sandbox = sinon.createSandbox(); + const msgContent = { + label: 'query2', + rateControl: { + type: 'maximum-rate', + opts: { startingTps: 20, finishingTps: 80 } + }, + workload: { + module:'./../queryByChannel.js' + }, + testRound:0, + txDuration:250, + totalClients: 1 + }; - sinon.assert.notCalled(updateSpy); - sinon.assert.calledOnce(sleepStub); - }); + sleepStub = sinon.stub(); - it('should sleep if no successful results are available', async () => { - let updateSpy = sandbox.spy(controller, 'updateOccurred'); - await controller.applyRateControl(null, 1, [], [{}, {}]); + const testMessage = new TestMessage('test', [], msgContent); + txnStats = new TransactionStatisticsCollector(); + controller = new MaxRate.createRateController(testMessage, txnStats, 0); - sinon.assert.notCalled(updateSpy); - sinon.assert.calledOnce(sleepStub); + sleepStub = sandbox.stub(controller, 'applySleepInterval'); + clock = sinon.useFakeTimers(); + }); + + afterEach( () => { + sandbox.restore(); + clock.restore(); }); - it('should sleep if no successful results are available', async () => { - let updateSpy = sandbox.spy(controller, 'updateOccurred'); - await controller.applyRateControl(null, 1, [], [{ucc: 12}, {}]); + it('should sleep if no completed transactions',async () => { + let exceededSpy = sandbox.spy(controller, 'exceededSampleInterval'); + await controller.applyRateControl(); - sinon.assert.notCalled(updateSpy); + sinon.assert.notCalled(exceededSpy); sinon.assert.calledOnce(sleepStub); }); - it('should initialize internal stats and tps maps on first pass', async () => { - let idx = 50; - let currentResults = []; - let item = { - succ: 5, - create: { - min: 100 - }, - final: { - last: 200 - } - }; - const resultStats = []; - resultStats.push(item); - resultStats.push(item); + it('should initialize internal stats on first pass', async () => { + sandbox.stub(controller, 'exceededSampleInterval').returns(false); - let exceededSampleIntervalSpy = sandbox.spy(controller, 'exceededSampleInterval'); - sandbox.stub(controller, 'updateOccurred').returns(false); - sandbox.stub(controller, 'retrieveIntervalTPS').returns(123); + txnStats.stats.txCounters.totalFinished = 500; - controller.init({}); - await controller.applyRateControl(null, idx, currentResults, resultStats); + controller.internalStats.lastUpdate.should.equal(0); + clock.tick(5); + await controller.applyRateControl(); // should have internal values - controller.statistics.previous.should.deep.equal(item); - controller.statistics.current.should.deep.equal(item); - controller.statistics.sampleStart.should.equal(100); - - controller.observedTPS.current.should.equal(123); - - // Should not have processed update - sinon.assert.notCalled(exceededSampleIntervalSpy); + controller.internalStats.lastUpdate.should.equal(5); }); - it('should ramp the driven TPS if current TPS > previous TPS', async () => { - let idx = 50; - let currentResults = []; - let item = { - succ: 5, - create: { - min: 100 - }, - final: { - last: 200 - } - }; - const resultStats = []; - resultStats.push(item); - resultStats.push(item); + it('should ramp the driven TPS if current TPS > previous TPS, including failed', async () => { - sandbox.stub(controller, 'updateOccurred').returns(true); + txnStats.stats.txCounters.totalFinished = 500; + txnStats.stats.txCounters.totalSuccessful = 400; + txnStats.stats.txCounters.totalFailed = 100; + txnStats.stats.latency.successful.total = 10; + txnStats.stats.latency.failed.total = 5; sandbox.stub(controller, 'exceededSampleInterval').returns(true); - sandbox.stub(controller, 'retrieveIntervalTPS').returns(10); + sandbox.stub(controller, 'retrieveIntervalTPS').returns(100); - controller.init({}); - controller.statistics.current = {}; - controller.observedTPS.current = 5; + controller.tpsSettings.current = 10; + await controller.applyRateControl(); - await controller.applyRateControl(null, idx, currentResults, resultStats); + controller.tpsSettings.current.should.equal(15); + controller.internalStats.currentCompletedTotal = 500; + controller.internalStats.currentElapsedTime = 15; + }); - controller.tpsSettings.current.should.equal(10); + it('should ramp the driven TPS if current TPS > previous TPS, not including failed', async () => { + txnStats.stats.txCounters.totalFinished = 500; + txnStats.stats.txCounters.totalSuccessful = 400; + txnStats.stats.txCounters.totalFailed = 100; + txnStats.stats.latency.successful.total = 10; + txnStats.stats.latency.failed.total = 5; + sandbox.stub(controller, 'exceededSampleInterval').returns(true); + sandbox.stub(controller, 'retrieveIntervalTPS').returns(100); + + controller.tpsSettings.current = 10; + controller.includeFailed = false; + await controller.applyRateControl(); + + controller.tpsSettings.current.should.equal(15); + controller.internalStats.currentCompletedTotal = 400; + controller.internalStats.currentElapsedTime = 10; }); - it('should drop the driven TPS and halve the step size if current TPS < previous TPS', async () => { - let idx = 50; - let currentResults = []; - let item = { - succ: 5, - create: { - min: 100 - }, - final: { - last: 200 - } - }; - const resultStats = []; - resultStats.push(item); - resultStats.push(item); + it('should drop the driven TPS and halve the step size if current TPS < previous TPS, including failed', async () => { + txnStats.stats.txCounters.totalFinished = 500; + txnStats.stats.txCounters.totalSuccessful = 400; + txnStats.stats.txCounters.totalFailed = 100; + txnStats.stats.latency.successful.total = 10; + txnStats.stats.latency.failed.total = 5; - sandbox.stub(controller, 'updateOccurred').returns(true); sandbox.stub(controller, 'exceededSampleInterval').returns(true); - sandbox.stub(controller, 'retrieveIntervalTPS').returns(10); + sandbox.stub(controller, 'retrieveIntervalTPS').returns(100); - controller.init({}); - controller.statistics.current = {}; - controller.observedTPS.current = 11; - controller.step = 5; - controller.tpsSettings.current = 20; - await controller.applyRateControl(null, idx, currentResults, resultStats); + controller.tpsSettings.current = 200; + controller.observedTPS.current = 250; + await controller.applyRateControl(); - controller.tpsSettings.current.should.equal(15); + controller.tpsSettings.current.should.equal(195); + controller.internalStats.currentCompletedTotal = 500; + controller.internalStats.currentElapsedTime = 15; controller.step.should.equal(2.5); }); - it('should drop the driven TPS only if current TPS < previous TPS and the step is below a threshold', async () => { - let idx = 50; - let currentResults = []; - let item = { - succ: 5, - create: { - min: 100 - }, - final: { - last: 200 - } - }; - const resultStats = []; - resultStats.push(item); - resultStats.push(item); + it('should drop the driven TPS and halve the step size if current TPS < previous TPS, not including failed', async () => { + txnStats.stats.txCounters.totalFinished = 500; + txnStats.stats.txCounters.totalSuccessful = 400; + txnStats.stats.txCounters.totalFailed = 100; + txnStats.stats.latency.successful.total = 10; + txnStats.stats.latency.failed.total = 5; - sandbox.stub(controller, 'updateOccurred').returns(true); sandbox.stub(controller, 'exceededSampleInterval').returns(true); - sandbox.stub(controller, 'retrieveIntervalTPS').returns(10); + sandbox.stub(controller, 'retrieveIntervalTPS').returns(100); - controller.init({}); - controller.statistics.current = {}; - controller.observedTPS.current = 11; - controller.step = 0.1; - controller.tpsSettings.current= 20; - await controller.applyRateControl(null, idx, currentResults, resultStats); + controller.tpsSettings.current = 200; + controller.observedTPS.current = 250; + controller.includeFailed = false; + await controller.applyRateControl(); - controller.tpsSettings.current.should.equal(19.9); - controller.step.should.equal(0.1); + controller.tpsSettings.current.should.equal(195); + controller.internalStats.currentCompletedTotal = 400; + controller.internalStats.currentElapsedTime = 10; + controller.step.should.equal(2.5); }); }); - describe('#updateOccurred', () => { - - let item = { - succ: 5, - create: { - min: 100 - }, - final: { - last: 200 - } - }; - const resultStats = []; - resultStats.push(item); - resultStats.push(item); - - it('should return true if the stored stats "create.min" differs from the passed', () => { - - let opts = {}; - let msg = {}; - let controller = new MaxRate.createRateController(opts); - controller.init(msg); - controller.statistics.current = { - create: { - min: 123 - } - }; + describe('#exceededSampleInterval', () => { - controller.updateOccurred(resultStats).should.equal(true); - }); + let sandbox, txnStats, controller, clock; - it('should return false if the stored stats "create.min" is the same as the passed', () => { + beforeEach(() => { - let opts = {}; - let msg = {}; - let controller = new MaxRate.createRateController(opts); - controller.init(msg); - controller.statistics.current = { - create: { - min: 100 - } - }; - controller.updateOccurred(resultStats).should.equal(false); - }); + sandbox = sinon.createSandbox(); - }); + clock = sinon.useFakeTimers(); - describe('#exceededSampleInterval', () => { + const msgContent = { + label: 'query2', + rateControl: { + type: 'maximum-rate', + opts: { startingTps: 20, finishingTps: 80 } + }, + workload: { + module:'./../queryByChannel.js' + }, + testRound:0, + txDuration:250, + totalClients: 1 + }; - let item = { - succ: 5, - create: { - min: 100 - }, - final: { - last: 2000 - } - }; - const resultStats = []; - resultStats.push(item); - resultStats.push(item); - - it('should return true if the sample time is less than the elapsed time', () => { - - let opts = {}; - let msg = {}; - let controller = new MaxRate.createRateController(opts); - controller.init(msg); - controller.statistics.sampleStart = 0; - - controller.exceededSampleInterval(resultStats).should.equal(true); + const testMessage = new TestMessage('test', [], msgContent); + txnStats = new TransactionStatisticsCollector(); + controller = new MaxRate.createRateController(testMessage, txnStats, 0); }); - it('should return false if the sample time is greater than the elapsed time', () => { + afterEach( () => { + sandbox.restore(); + clock.restore(); + }); - let opts = {}; - let msg = {}; - let controller = new MaxRate.createRateController(opts); - controller.init(msg); - controller.statistics.sampleStart = 1999; - controller.exceededSampleInterval(resultStats).should.equal(false); + it('should return true if the elapsed time exceeds the configured sample interval', () => { + clock.tick(100); + controller.internalStats.lastUpdate = 1; + controller.sampleInterval = 1; + controller.exceededSampleInterval().should.equal(true); + }); + + it('should return false if the elapsed time is less than the configured sample interval', () => { + clock.tick(100); + controller.internalStats.lastUpdate = 1; + controller.sampleInterval = 10000; + controller.exceededSampleInterval().should.equal(false); }); }); describe('#retrieveIntervalTPS', () => { - let item = { - succ: 50, - fail: 50, - create: { - min: 10 - }, - final: { - last: 20 - } - }; - const resultStats = []; - resultStats.push(item); - resultStats.push(item); - - it('should return the TPS from the interval including failed transactions', () => { - - let opts = { - includeFailed: true + let sandbox, txnStats, controller; + + beforeEach(() => { + + sandbox = sinon.createSandbox(); + + const msgContent = { + label: 'query2', + rateControl: { + type: 'maximum-rate', + opts: { startingTps: 20, finishingTps: 80 } + }, + workload: { + module:'./../queryByChannel.js' + }, + testRound:0, + txDuration:250, + totalClients: 1 }; - let msg = {}; - let controller = new MaxRate.createRateController(opts); - controller.init(msg); - controller.statistics.sampleStart = 0; - controller.retrieveIntervalTPS(resultStats).should.equal(10); + const testMessage = new TestMessage('test', [], msgContent); + txnStats = new TransactionStatisticsCollector(); + controller = new MaxRate.createRateController(testMessage, txnStats, 0); }); - it('should return the TPS from the interval excluding failed transactions', () => { + afterEach( () => { + sandbox.restore(); + }); - let opts = { - includeFailed: false - }; - let msg = {}; - let controller = new MaxRate.createRateController(opts); - controller.init(msg); - controller.statistics.sampleStart = 0; + it('should return the TPS from internalStats', () => { - controller.retrieveIntervalTPS(resultStats).should.equal(10); + controller.internalStats.currentCompletedTotal = 10; + controller.internalStats.currentElapsedTime = 5; + controller.retrieveIntervalTPS().should.equal(2000); }); }); describe('#applySleepInterval', () => { - it('should apply the global TPS setting as a sleep interval', () => { + let sandbox; + let sleepStub; + let controller; + let txnStats; + + beforeEach(() => { + + sandbox = sinon.createSandbox(); + const msgContent = { + label: 'query2', + rateControl: { + type: 'maximum-rate', + opts: { startingTps: 20, finishingTps: 80 } + }, + workload: { + module:'./../queryByChannel.js' + }, + testRound:0, + txDuration:250, + totalClients: 1 + }; + + sleepStub = sinon.stub(); + + const testMessage = new TestMessage('test', [], msgContent); + txnStats = new TransactionStatisticsCollector(); - let opts = {}; - let msg = {}; - let sleepStub = sinon.stub(); MaxRate.__set__('Sleep', sleepStub); - let controller = new MaxRate.createRateController(opts); - controller.init(msg); - controller.statistics.sampleStart = 0; + controller = new MaxRate.createRateController(testMessage, txnStats, 0); + }); + + afterEach( () => { + sandbox.restore(); + }); - controller.applySleepInterval(); + it('should apply the global TPS setting as a sleep interval', async () => { + await controller.applySleepInterval(); // 200 = 1000/default sinon.assert.calledOnceWithExactly(sleepStub, 200); }); diff --git a/packages/caliper-core/test/worker/rate-control/noRate.js b/packages/caliper-core/test/worker/rate-control/noRate.js index 178092346c..9e082e4c29 100644 --- a/packages/caliper-core/test/worker/rate-control/noRate.js +++ b/packages/caliper-core/test/worker/rate-control/noRate.js @@ -16,21 +16,41 @@ const rewire = require('rewire'); const NoRate = rewire('../../../lib/worker/rate-control/noRate'); +const TestMessage = require('../../../lib/common/messages/testMessage'); +const TransactionStatisticsCollector = require('../../../lib/common/core/transaction-statistics-collector'); const chai = require('chai'); chai.should(); const sinon = require('sinon'); const assert = require('assert'); -describe ('noRate controller implementation', () => { - let controller; +describe('noRate controller implementation', () => { + + describe('#constructor', () => { + + let controller, testMessage; + beforeEach( () => { + const msgContent = { + label: 'query2', + rateControl: { + type: 'linear-rate', + opts: {} + }, + workload: { + module:'./../queryByChannel.js' + }, + testRound:0, + txDuration:250, + totalClients:2 + }; + testMessage = new TestMessage('test', [], msgContent); + }); + - describe('#init', () => { it ('should throw an error if a value for the number of transactions is set', async () => { try { - controller = new NoRate.createRateController({}); - const msg = {numb: 5}; - await controller.init(msg); + testMessage.content.numb = 6; + controller = new NoRate.createRateController(testMessage, {}, 0); assert.fail(null, null, 'Exception expected'); } catch (error) { if (error.constructor.name === 'AssertionError') { @@ -41,29 +61,52 @@ describe ('noRate controller implementation', () => { }); it ('should set the sleep time based on the length of the round in seconds', () => { - let msg = {txDuration: 100}; - controller = new NoRate.createRateController({}); - controller.init(msg); + testMessage.content.txDuration = 6; + controller = new NoRate.createRateController(testMessage, {}, 0); - let sleepTime = msg.txDuration * 1000; - controller.sleepTime.should.equal(sleepTime); + controller.sleepTime.should.equal(6000); }); }); describe('#applyRateControl', () => { - it('should sleep for the set sleep time', () => { - let sleepStub = sinon.stub(); - let msg = {txDuration: 100}; - NoRate.__set__('Util.sleep', sleepStub); - controller = new NoRate.createRateController({}); + let controller, sleepStub, msgContent, clock; + + beforeEach(() => { + msgContent = { + label: 'query2', + rateControl: { + type: 'linear-feedback-rate', + opts: { startingTps: 20, finishingTps: 80 } + }, + workload: { + module:'./../queryByChannel.js' + }, + testRound:0, + txDuration:250, + totalClients:2 + }; + + clock = sinon.useFakeTimers(); + + sleepStub = sinon.stub(); + NoRate.__set__('Sleep', sleepStub); + }); + + afterEach(() => { + clock.restore(); + }); + + it('should sleep for the set sleep time', () => { + const txnStats = new TransactionStatisticsCollector(); - controller.sleepTime = msg.txDuration * 1000; - let sleepTime = controller.sleepTime; + const testMessage = new TestMessage('test', [], msgContent); + testMessage.content.txDuration = 5; + controller = new NoRate.createRateController(testMessage, txnStats, 0); - controller.applyRateControl(null, null, null, null); + controller.applyRateControl(); sinon.assert.calledOnce(sleepStub); - sinon.assert.calledWith(sleepStub, sleepTime); + sinon.assert.calledWith(sleepStub, 5000); }); }); }); diff --git a/packages/caliper-tests-integration/besu_tests/phase1/benchconfig.yaml b/packages/caliper-tests-integration/besu_tests/phase1/benchconfig.yaml index 5d74c188d3..896cfce76a 100644 --- a/packages/caliper-tests-integration/besu_tests/phase1/benchconfig.yaml +++ b/packages/caliper-tests-integration/besu_tests/phase1/benchconfig.yaml @@ -17,6 +17,3 @@ test: workers: type: local number: 1 -observer: - interval: 1 - type: local diff --git a/packages/caliper-tests-integration/besu_tests/phase2/benchconfig.yaml b/packages/caliper-tests-integration/besu_tests/phase2/benchconfig.yaml index fed5c13bf2..3f97afb847 100644 --- a/packages/caliper-tests-integration/besu_tests/phase2/benchconfig.yaml +++ b/packages/caliper-tests-integration/besu_tests/phase2/benchconfig.yaml @@ -47,6 +47,3 @@ test: accountPhasePrefix: *prefix numberOfAccounts: *accounts money: 100 -observer: - interval: 1 - type: local diff --git a/packages/caliper-tests-integration/besu_tests/phase3/benchconfig.yaml b/packages/caliper-tests-integration/besu_tests/phase3/benchconfig.yaml index 2cd159c98b..6b520d893d 100644 --- a/packages/caliper-tests-integration/besu_tests/phase3/benchconfig.yaml +++ b/packages/caliper-tests-integration/besu_tests/phase3/benchconfig.yaml @@ -47,6 +47,3 @@ test: accountPhasePrefix: *prefix numberOfAccounts: *accounts money: 100 -observer: - interval: 1 - type: local diff --git a/packages/caliper-tests-integration/ethereum_tests/benchconfig.yaml b/packages/caliper-tests-integration/ethereum_tests/benchconfig.yaml index c9cbb214ae..37118c268f 100644 --- a/packages/caliper-tests-integration/ethereum_tests/benchconfig.yaml +++ b/packages/caliper-tests-integration/ethereum_tests/benchconfig.yaml @@ -47,6 +47,4 @@ test: accountPhasePrefix: *prefix numberOfAccounts: *accounts money: 100 -observer: - interval: 1 - type: local + diff --git a/packages/caliper-tests-integration/fabric_docker_distributed_tests/benchconfig.yaml b/packages/caliper-tests-integration/fabric_docker_distributed_tests/benchconfig.yaml index 2e0432307d..82722e8196 100644 --- a/packages/caliper-tests-integration/fabric_docker_distributed_tests/benchconfig.yaml +++ b/packages/caliper-tests-integration/fabric_docker_distributed_tests/benchconfig.yaml @@ -28,8 +28,3 @@ test: rateControl: { type: 'linear-rate', opts: { startingTps: 5, finishingTps: 10 } } workload: module: ./query.js -observer: - interval: 5 - type: local -monitor: - type: ['none'] diff --git a/packages/caliper-tests-integration/fabric_docker_local_tests/benchconfig.yaml b/packages/caliper-tests-integration/fabric_docker_local_tests/benchconfig.yaml index 2e0432307d..82722e8196 100644 --- a/packages/caliper-tests-integration/fabric_docker_local_tests/benchconfig.yaml +++ b/packages/caliper-tests-integration/fabric_docker_local_tests/benchconfig.yaml @@ -28,8 +28,3 @@ test: rateControl: { type: 'linear-rate', opts: { startingTps: 5, finishingTps: 10 } } workload: module: ./query.js -observer: - interval: 5 - type: local -monitor: - type: ['none'] diff --git a/packages/caliper-tests-integration/fabric_tests/phase2/benchconfig.yaml b/packages/caliper-tests-integration/fabric_tests/phase2/benchconfig.yaml index d7bb5b38b6..997a6926d3 100644 --- a/packages/caliper-tests-integration/fabric_tests/phase2/benchconfig.yaml +++ b/packages/caliper-tests-integration/fabric_tests/phase2/benchconfig.yaml @@ -33,11 +33,9 @@ test: rateControl: { type: 'linear-rate', opts: { startingTps: 10, finishingTps: 20 } } workload: module: ./../query.js -observer: - interval: 1 - type: local -monitor: - interval: 1 - type: ['process'] - process: - processes: [{ command: 'node', arguments: 'fabricClientWorker.js', multiOutput: 'avg' }] +monitors: + resource: + - module: process + options: + interval: 3 + processes: [{ command: 'node', arguments: 'caliper.js', multiOutput: 'avg' }] diff --git a/packages/caliper-tests-integration/fabric_tests/phase3/benchconfig.yaml b/packages/caliper-tests-integration/fabric_tests/phase3/benchconfig.yaml index 120ff6fd8b..5fbaab022b 100644 --- a/packages/caliper-tests-integration/fabric_tests/phase3/benchconfig.yaml +++ b/packages/caliper-tests-integration/fabric_tests/phase3/benchconfig.yaml @@ -33,13 +33,13 @@ test: rateControl: { type: 'linear-rate', opts: { startingTps: 10, finishingTps: 20 } } workload: module: ./../query.js -observer: - interval: 1 - type: local -monitor: - interval: 1 - type: ['process', 'docker'] - process: - processes: [{ command: 'node', arguments: 'fabricClientWorker.js', multiOutput: 'avg' }] - docker: +monitors: + resource: + - module: process + options: + interval: 3 + processes: [{ command: 'node', arguments: 'caliper.js', multiOutput: 'avg' }] + - module: docker + options: + interval: 4 containers: ['peer0.org1.example.com', 'peer0.org2.example.com', 'orderer0.example.com', 'orderer1.example.com'] diff --git a/packages/caliper-tests-integration/fabric_tests/phase4/benchconfig.yaml b/packages/caliper-tests-integration/fabric_tests/phase4/benchconfig.yaml index 92ea3f4427..f1c8f88192 100644 --- a/packages/caliper-tests-integration/fabric_tests/phase4/benchconfig.yaml +++ b/packages/caliper-tests-integration/fabric_tests/phase4/benchconfig.yaml @@ -42,6 +42,38 @@ test: rateControl: { type: 'linear-rate', opts: { startingTps: 5, finishingTps: 10 } } workload: module: ./../queryByChannel.js -observer: - interval: 1 - type: local + +monitors: + transaction: + - module: logging + options: + loggerModuleName: txinfo + messageLevel: info + - module: prometheus-push + options: + interval: 5 + push_url: "http://localhost:9091" + resource: + - module: prometheus + options: + interval: 5 + url: "http://localhost:9090" + metrics: + ignore: [prometheus, pushGateway, cadvisor, grafana, node-exporter] + include: + Endorse Time (s): + query: rate(endorser_propsal_duration_sum{chaincode="marbles:v0"}[1m])/rate(endorser_propsal_duration_count{chaincode="marbles:v0"}[1m]) + step: 1 + label: instance + statistic: avg + Max Memory (MB): + query: sum(container_memory_rss{name=~".+"}) by (name) + step: 10 + label: name + statistic: max + multiplier: 0.000001 + charting: + polar: + metrics: [Max Memory (MB)] + bar: + metrics: [all] diff --git a/packages/caliper-tests-integration/fabric_tests/phase5/benchconfig.yaml b/packages/caliper-tests-integration/fabric_tests/phase5/benchconfig.yaml index d15493c5b9..66ee57a478 100644 --- a/packages/caliper-tests-integration/fabric_tests/phase5/benchconfig.yaml +++ b/packages/caliper-tests-integration/fabric_tests/phase5/benchconfig.yaml @@ -18,39 +18,32 @@ test: type: local number: 2 rounds: - - label: init1 - txNumber: 25 - rateControl: { type: 'fixed-rate', opts: { tps: 5 } } - workload: - module: ./../init.js - arguments: - marblePrefix: marbles_phase_5 - label: query1 - txNumber: 25 - rateControl: { type: 'linear-rate', opts: { startingTps: 5, finishingTps: 10 } } + txNumber: 50 + rateControl: { type: 'fixed-rate', opts: { tps: 3 } } workload: module: ./../query.js - - label: init2 - txNumber: 25 - rateControl: { type: 'fixed-rate', opts: { tps: 5 } } - workload: - module: ./../initByChannel.js - arguments: - marblePrefix: marbles_phase_5 - label: query2 - txNumber: 25 - rateControl: { type: 'linear-rate', opts: { startingTps: 5, finishingTps: 10 } } + txDuration: 20 + rateControl: { type: 'fixed-load', opts: { startTps: 10, transactionLoad: 20 } } workload: module: ./../queryByChannel.js -observer: - interval: 1 - type: prometheus -monitor: - interval: 1 - type: ['prometheus'] - prometheus: - url: "http://localhost:9090" + +monitors: + transaction: + - module: logging + options: + loggerModuleName: txinfo + messageLevel: info + - module: prometheus-push + options: + interval: 5 push_url: "http://localhost:9091" + resource: + - module: prometheus + options: + interval: 5 + url: "http://localhost:9090" metrics: ignore: [prometheus, pushGateway, cadvisor, grafana, node-exporter] include: @@ -70,3 +63,11 @@ monitor: metrics: [Max Memory (MB)] bar: metrics: [all] + - module: process + options: + interval: 3 + processes: [{ command: 'node', arguments: 'caliper.js', multiOutput: 'avg' }] + - module: docker + options: + interval: 4 + containers: ['peer0.org1.example.com', 'peer0.org2.example.com', 'orderer0.example.com', 'orderer1.example.com'] diff --git a/packages/caliper-tests-integration/fabric_tests/phase6/benchconfig.yaml b/packages/caliper-tests-integration/fabric_tests/phase6/benchconfig.yaml index 4b084b7982..88d8e209b3 100644 --- a/packages/caliper-tests-integration/fabric_tests/phase6/benchconfig.yaml +++ b/packages/caliper-tests-integration/fabric_tests/phase6/benchconfig.yaml @@ -42,15 +42,21 @@ test: rateControl: { type: 'linear-rate', opts: { startingTps: 5, finishingTps: 10 } } workload: module: ./../queryByChannel.js -observer: - interval: 1 - type: prometheus -monitor: - interval: 1 - type: ['prometheus'] - prometheus: - url: "http://localhost:9090" +monitors: + transaction: + - module: logging + options: + loggerModuleName: txinfo + messageLevel: info + - module: prometheus-push + options: + interval: 5 push_url: "http://localhost:9091" + resource: + - module: prometheus + options: + interval: 5 + url: "http://localhost:9090" metrics: ignore: [prometheus, pushGateway, cadvisor, grafana, node-exporter] include: diff --git a/packages/caliper-tests-integration/fabric_tests/phase7/benchconfig.yaml b/packages/caliper-tests-integration/fabric_tests/phase7/benchconfig.yaml index fd5d049182..eb610576e9 100644 --- a/packages/caliper-tests-integration/fabric_tests/phase7/benchconfig.yaml +++ b/packages/caliper-tests-integration/fabric_tests/phase7/benchconfig.yaml @@ -37,11 +37,3 @@ test: rateControl: { type: 'linear-rate', opts: { startingTps: 10, finishingTps: 20 } } workload: module: ./../query.js -observer: - interval: 1 - type: local -monitor: - interval: 1 - type: ['process'] - process: - processes: [{ command: 'node', arguments: 'fabricClientWorker.js', multiOutput: 'avg' }] diff --git a/packages/caliper-tests-integration/fisco-bcos_tests/benchconfig.yaml b/packages/caliper-tests-integration/fisco-bcos_tests/benchconfig.yaml index 63437d870d..803a9b1109 100644 --- a/packages/caliper-tests-integration/fisco-bcos_tests/benchconfig.yaml +++ b/packages/caliper-tests-integration/fisco-bcos_tests/benchconfig.yaml @@ -28,6 +28,3 @@ test: rateControl: { type: 'linear-rate', opts: { startingTps: 100, finishingTps: 200 } } workload: module: ./get.js -observer: - interval: 1 - type: local diff --git a/packages/caliper-tests-integration/generator_tests/fabric/caliper.yaml b/packages/caliper-tests-integration/generator_tests/fabric/caliper.yaml index bcd0e9228b..d1f8070139 100644 --- a/packages/caliper-tests-integration/generator_tests/fabric/caliper.yaml +++ b/packages/caliper-tests-integration/generator_tests/fabric/caliper.yaml @@ -41,7 +41,7 @@ caliper: target: console enabled: true options: - level: debug + level: info file: target: file enabled: false diff --git a/packages/caliper-tests-integration/generator_tests/fabric/run.sh b/packages/caliper-tests-integration/generator_tests/fabric/run.sh index 03815734f8..6784bd6cfe 100755 --- a/packages/caliper-tests-integration/generator_tests/fabric/run.sh +++ b/packages/caliper-tests-integration/generator_tests/fabric/run.sh @@ -57,7 +57,7 @@ fi cd ${DIR} # Run benchmark generator not using generator defaults rm -r myWorkspace/benchmarks -${GENERATOR_METHOD} -- --workspace 'myWorkspace' --contractId 'mymarbles' --contractVersion 'v0' --contractFunction 'queryMarblesByOwner' --contractArguments '["Alice"]' --workers 3 --benchmarkName 'A name for the marbles benchmark' --benchmarkDescription 'A description for the marbles benchmark' --label 'A label for the round' --rateController 'fixed-rate' --txType 'txDuration' --txDuration 30 +${GENERATOR_METHOD} -- --workspace 'myWorkspace' --contractId 'mymarbles' --contractVersion 'v0' --contractFunction 'queryMarblesByOwner' --contractArguments '["Alice"]' --workers 2 --benchmarkName 'A name for the marbles benchmark' --benchmarkDescription 'A description for the marbles benchmark' --label 'A label for the round' --rateController 'fixed-rate' --txType 'txDuration' --txDuration 10 # Run benchmark test cd ../ diff --git a/packages/caliper-tests-integration/iroha_tests/benchconfig.yaml b/packages/caliper-tests-integration/iroha_tests/benchconfig.yaml index 1db31f5e33..c903fbcc74 100644 --- a/packages/caliper-tests-integration/iroha_tests/benchconfig.yaml +++ b/packages/caliper-tests-integration/iroha_tests/benchconfig.yaml @@ -44,6 +44,3 @@ test: txnPerBatch: 10 numberOfAccounts: *accounts money: '10.00' -observer: - interval: 1 - type: local diff --git a/packages/caliper-tests-integration/sawtooth_tests/benchconfig.yaml b/packages/caliper-tests-integration/sawtooth_tests/benchconfig.yaml index ff39f90fbd..f9a3a87444 100644 --- a/packages/caliper-tests-integration/sawtooth_tests/benchconfig.yaml +++ b/packages/caliper-tests-integration/sawtooth_tests/benchconfig.yaml @@ -35,7 +35,3 @@ test: module: ./query.js arguments: accounts: 30 -observer: - interval: 1 - type: local - diff --git a/packages/generator-caliper/generators/benchmark/index.js b/packages/generator-caliper/generators/benchmark/index.js index 1a17b139a9..e7935159a8 100644 --- a/packages/generator-caliper/generators/benchmark/index.js +++ b/packages/generator-caliper/generators/benchmark/index.js @@ -18,7 +18,8 @@ const Generator = require('yeoman-generator'); const camelcase = require('camelcase'); const defaultTxValue = 50; -const defaultClientValue = 5; +const defaultTxDuration = 20; +const defaultWorkerCount = 1; const answersObject = {}; let promptAnswers; @@ -86,7 +87,7 @@ module.exports = class extends Generator { type: 'number', name: 'workers', message: 'How many workers would you like to have?', - default: defaultClientValue, + default: defaultWorkerCount, when: () => !this.options.workers }], roundQuestions: [{ @@ -119,7 +120,7 @@ module.exports = class extends Generator { type: 'number', name: 'txDuration', message: 'How long would you like the round to last?', - default: defaultTxValue, + default: defaultTxDuration, when: () => !this.options.txDuration }], txNumberQuestion : [{ @@ -135,7 +136,7 @@ module.exports = class extends Generator { const clientAnswer = await this.prompt(benchmarkQuestions.clientQuestions); if (isNaN(parseFloat(this.options.workers)) && isNaN(parseFloat(clientAnswer.workers))) { - this.log(`Error: Not a valid input. Using default client value of ${defaultClientValue}.`); + this.log(`Error: Not a valid input. Using default client value of ${defaultWorkerCount}.`); } if (this.options.workers < 0 || clientAnswer.workers < 0) { this.log(`Error: Negative values not accepted. Defaulting to ${Math.abs(clientAnswer.workers)}.`); @@ -147,7 +148,7 @@ module.exports = class extends Generator { if (roundAnswers.txType === 'txDuration') { txValueAnswer = await this.prompt(benchmarkQuestions.txDurationQuestion); if (isNaN(parseFloat(txValueAnswer.txDuration))) { - this.log(`Error: Not a valid input. Using default txDuration value of ${defaultTxValue}.`); + this.log(`Error: Not a valid input. Using default txDuration value of ${defaultTxDuration}.`); } if (txValueAnswer.txDuration < 0) { this.log(`Error: Negative values not accepted. Defaulting to ${Math.abs(txValueAnswer.txDuration)}.`); @@ -211,7 +212,7 @@ module.exports = class extends Generator { answersObject.contractId = promptAnswers.contractId; if (isNaN(promptAnswers.workers)) { - answersObject.workers = defaultClientValue; + answersObject.workers = defaultWorkerCount; } else if (promptAnswers.workers < 0) { answersObject.workers = Math.abs(promptAnswers.workers); } @@ -221,7 +222,7 @@ module.exports = class extends Generator { if (promptAnswers.txType === 'txDuration') { if (isNaN(promptAnswers.txDuration)) { - answersObject.txValue = defaultTxValue; + answersObject.txValue = defaultTxDuration; } else if (promptAnswers.txDuration < 0) { answersObject.txValue = Math.abs(promptAnswers.txDuration); } else { diff --git a/packages/generator-caliper/generators/benchmark/templates/config.yaml b/packages/generator-caliper/generators/benchmark/templates/config.yaml index cb76957ea1..267773d0c4 100644 --- a/packages/generator-caliper/generators/benchmark/templates/config.yaml +++ b/packages/generator-caliper/generators/benchmark/templates/config.yaml @@ -23,7 +23,7 @@ test: - label: <%= label %> contractId: <%= contractId %> <%= txType %>: <%= txValue %> - rateControl: + rateControl: type: <%= rateController %> opts: { <%= opts %> } workload: @@ -31,6 +31,3 @@ test: arguments: contractId: <%= contractId %> contractVersion: <%= contractVersion %> -monitor: - type: - - none diff --git a/packages/generator-caliper/test/benchmark/config.js b/packages/generator-caliper/test/benchmark/config.js index 0c31ba933a..36510ed008 100644 --- a/packages/generator-caliper/test/benchmark/config.js +++ b/packages/generator-caliper/test/benchmark/config.js @@ -67,9 +67,6 @@ describe ('benchmark configuration generator', () => { } } ] - }, - monitor: { - type: [ 'none' ] } }; @@ -187,7 +184,7 @@ describe ('benchmark configuration generator', () => { const config = yaml.safeLoad(fs.readFileSync(tmpConfigPath),'utf8'); const configStr = JSON.stringify(config); - const fileContains = configStr.includes('"workers":{"type":"local","number":5'); + const fileContains = configStr.includes('"workers":{"type":"local","number":1'); fileContains.should.equal(true); }); @@ -234,7 +231,7 @@ describe ('benchmark configuration generator', () => { const config = yaml.safeLoad(fs.readFileSync(tmpConfigPath),'utf8'); const configStr = JSON.stringify(config); - const fileContains = configStr.includes('"txDuration":50'); + const fileContains = configStr.includes('"txDuration":20'); fileContains.should.equal(true); });