diff --git a/packages/caliper-burrow/package.json b/packages/caliper-burrow/package.json index 6d8e7a3ef..42ce8bf2b 100644 --- a/packages/caliper-burrow/package.json +++ b/packages/caliper-burrow/package.json @@ -1,5 +1,6 @@ { "name": "@hyperledger/caliper-burrow", + "description": "Hyperledger Burrow adaptor for Caliper, enabling the running of a performance benchmarks that interact with Burrow", "version": "0.1.0", "repository": { "type": "git", diff --git a/packages/caliper-cli/lib/zooclient.js b/packages/caliper-cli/lib/zooclient.js deleted file mode 100644 index 2b9b90087..000000000 --- a/packages/caliper-cli/lib/zooclient.js +++ /dev/null @@ -1,26 +0,0 @@ -/* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -'use strict'; - -exports.command = 'zooclient '; -exports.desc = 'Caliper zookeeper client command'; -exports.builder = function (yargs) { - // apply commands in subdirectories - return yargs.demandCommand(1, 'Incorrect command. Please see the list of commands above, or enter "caliper zooclient --help".') - .commandDir('zooclient'); -}; -exports.handler = function (argv) {}; - -module.exports.StartZooClient = require('./zooclient/lib/startZooClient'); diff --git a/packages/caliper-cli/lib/zooclient/lib/startZooClient.js b/packages/caliper-cli/lib/zooclient/lib/startZooClient.js deleted file mode 100644 index 2a60e54b6..000000000 --- a/packages/caliper-cli/lib/zooclient/lib/startZooClient.js +++ /dev/null @@ -1,65 +0,0 @@ -/* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -'use strict'; - -const {CaliperUtils, CaliperZooClient, ConfigUtil} = require('@hyperledger/caliper-core'); -const chalk = require('chalk'); -const cmdUtil = require('../../utils/cmdutils'); -const path = require('path'); -const fs = require('fs'); - -/** - * Star a zoo client - */ -class StartZooClient { - - /** - * Command process for start zookeeper command - * @param {string} argv argument list from caliper command - */ - static async handler(argv) { - let blockchainConfigFile = ConfigUtil.get(ConfigUtil.keys.NetworkConfig, undefined); - let workspace = ConfigUtil.get(ConfigUtil.keys.Workspace, './'); - - // Workspace is expected to be the root location of working folders - workspace = path.resolve(workspace); - blockchainConfigFile = path.isAbsolute(blockchainConfigFile) ? blockchainConfigFile : path.join(workspace, blockchainConfigFile); - - if(!blockchainConfigFile || !fs.existsSync(blockchainConfigFile)) { - throw(new Error(`Network configuration file "${blockchainConfigFile || 'UNSET'}" does not exist`)); - } - - let blockchainType = ''; - let networkObject = CaliperUtils.parseYaml(blockchainConfigFile); - if (networkObject.hasOwnProperty('caliper') && networkObject.caliper.hasOwnProperty('blockchain')) { - blockchainType = networkObject.caliper.blockchain; - } else { - throw new Error('The configuration file [' + blockchainConfigFile + '] is missing its "caliper.blockchain" attribute'); - } - - try { - cmdUtil.log(chalk.blue.bold('Starting zookeeper client of type ' + blockchainType)); - const {ClientFactory} = require('@hyperledger/caliper-' + blockchainType); - const clientFactory = new ClientFactory(blockchainConfigFile, workspace); - - const zooClient = new CaliperZooClient(ConfigUtil.get(ConfigUtil.keys.ZooAddress, undefined), clientFactory, workspace); - zooClient.start(); - } catch (err) { - throw err; - } - } -} - -module.exports = StartZooClient; diff --git a/packages/caliper-cli/lib/zooclient/startZooClientCommand.js b/packages/caliper-cli/lib/zooclient/startZooClientCommand.js deleted file mode 100644 index 967c8d2b3..000000000 --- a/packages/caliper-cli/lib/zooclient/startZooClientCommand.js +++ /dev/null @@ -1,52 +0,0 @@ -/* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -'use strict'; - -const Start = require ('./lib/startZooClient'); - -// enforces singletons -const checkFn = (argv, options) => { - - ['caliper-zooaddress','caliper-networkconfig','caliper-workspace'].forEach((e)=>{ - if (Array.isArray(argv[e])){ - throw new Error(`Option ${e} can only be specified once`); - } - }); - - return true; -}; -module.exports._checkFn = checkFn; -module.exports.command = 'start'; -module.exports.describe = 'Start a zookeeper client on a provided host:port using provided blockchain configuration and a workspace location'; -module.exports.builder = function (yargs){ - yargs.options({ - 'caliper-zooaddress' : {describe: 'zookeeper address', type: 'string' }, - 'caliper-networkconfig' : {describe:'Path to the blockchain configuration file that contains information required to interact with the SUT', type: 'string'}, - 'caliper-workspace' : {describe:'Workspace directory that contains all configuration information', type: 'string'} - }); - yargs.usage('caliper zooclient start --caliper-workspace ~/myCaliperProject --caliper-zooaddress : --caliper-networkconfig my-sut-config.yaml'); - - // enforce the option after these options - yargs.requiresArg(['caliper-zooaddress','caliper-networkconfig','caliper-workspace']); - - // enforce singletons - yargs.check(checkFn); - - return yargs; -}; - -module.exports.handler = (argv) => { - return argv.thePromise = Start.handler(argv); -}; diff --git a/packages/caliper-cli/lib/zooservice.js b/packages/caliper-cli/lib/zooservice.js deleted file mode 100644 index 218ff69b1..000000000 --- a/packages/caliper-cli/lib/zooservice.js +++ /dev/null @@ -1,27 +0,0 @@ -/* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -'use strict'; - -exports.command = 'zooservice '; -exports.desc = 'Caliper zookeeper service command'; -exports.builder = function (yargs) { - // apply commands in subdirectories - return yargs.demandCommand(1, 'Incorrect command. Please see the list of commands above, or enter "caliper zooservice --help".') - .commandDir('zooservice'); -}; -exports.handler = function (argv) {}; - -module.exports.StartZooService = require('./zooservice/lib/startZooService'); -module.exports.StopZooService = require('./zooservice/lib/stopZooService'); diff --git a/packages/caliper-cli/lib/zooservice/lib/startZooService.js b/packages/caliper-cli/lib/zooservice/lib/startZooService.js deleted file mode 100644 index 9a0a64510..000000000 --- a/packages/caliper-cli/lib/zooservice/lib/startZooService.js +++ /dev/null @@ -1,69 +0,0 @@ -/* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -'use strict'; - -const {ConfigUtil} = require('@hyperledger/caliper-core'); -const childProcess = require('child_process'); -const exec = childProcess.exec; - -const chalk = require('chalk'); -const cmdUtil = require('../../utils/cmdutils'); - -/** - * Start the zoo service - */ -class StartZooService { - - /** - * Command process for run benchmark command - * @param {string} argv argument list from caliper command - */ - static async handler(argv) { - let cmd = 'docker-compose -f '; - if (ConfigUtil.get(ConfigUtil.keys.ZooConfig, undefined)){ - cmd += argv.config + ' up'; - } else { - cmdUtil.log(chalk.blue.bold('Using default configuration file')); - cmd += __dirname + '/zookeeper-service.yaml up'; - } - - cmdUtil.log(chalk.blue.bold('Starting zookeeper service ......')); - await StartZooService.execAsync(cmd); - cmdUtil.log(chalk.blue.bold('Start zookeeper service successful')); - } - - /** - * Executes the given command asynchronously. - * @param {string} command The command to execute through a newly spawn shell. - * @return {Promise} The return promise is resolved upon the successful execution of the command, or rejected with an Error instance. - * @async - */ - static execAsync(command) { - return new Promise((resolve, reject) => { - cmdUtil.log(chalk.blue.bold(`Executing command: ${command}`)); - let child = exec(command, (err, stdout, stderr) => { - if (err) { - cmdUtil.log(chalk.red.bold(`Unsuccessful command execution. Error code: ${err.code}. Terminating signal: ${err.signal}`)); - return reject(err); - } - return resolve(); - }); - child.stdout.pipe(process.stdout); - child.stderr.pipe(process.stderr); - }); - } -} - -module.exports = StartZooService; diff --git a/packages/caliper-cli/lib/zooservice/lib/stopZooService.js b/packages/caliper-cli/lib/zooservice/lib/stopZooService.js deleted file mode 100644 index f7873967a..000000000 --- a/packages/caliper-cli/lib/zooservice/lib/stopZooService.js +++ /dev/null @@ -1,69 +0,0 @@ -/* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -'use strict'; - -const {ConfigUtil} = require('@hyperledger/caliper-core'); -const childProcess = require('child_process'); -const exec = childProcess.exec; - -const chalk = require('chalk'); -const cmdUtil = require('../../utils/cmdutils'); - -/** - * Stop the zoo service - */ -class StopZooService { - - /** - * Command process for run benchmark command - * @param {string} argv argument list from caliper command - */ - static async handler(argv) { - let cmd = 'docker-compose -f '; - if (ConfigUtil.get(ConfigUtil.keys.ZooConfig, undefined)){ - cmd += argv.config + ' down'; - } else { - cmdUtil.log(chalk.blue.bold('Using default configuration file')); - cmd += __dirname + '/zookeeper-service.yaml down'; - } - - cmdUtil.log(chalk.blue.bold('Stopping zookeeper service ......')); - await StopZooService.execAsync(cmd); - cmdUtil.log(chalk.blue.bold('Stop zookeeper service successful')); - } - - /** - * Executes the given command asynchronously. - * @param {string} command The command to execute through a newly spawn shell. - * @return {Promise} The return promise is resolved upon the successful execution of the command, or rejected with an Error instance. - * @async - */ - static execAsync(command) { - return new Promise((resolve, reject) => { - cmdUtil.log(chalk.blue.bold(`Executing command: ${command}`)); - let child = exec(command, (err, stdout, stderr) => { - if (err) { - cmdUtil.log(chalk.red.bold(`Unsuccessful command execution. Error code: ${err.code}. Terminating signal: ${err.signal}`)); - return reject(err); - } - return resolve(); - }); - child.stdout.pipe(process.stdout); - child.stderr.pipe(process.stderr); - }); - } -} - -module.exports = StopZooService; diff --git a/packages/caliper-cli/lib/zooservice/lib/zookeeper-service.yaml b/packages/caliper-cli/lib/zooservice/lib/zookeeper-service.yaml deleted file mode 100644 index 4fa7594c3..000000000 --- a/packages/caliper-cli/lib/zooservice/lib/zookeeper-service.yaml +++ /dev/null @@ -1,46 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -version: '3.1' - -services: - zoo1: - image: zookeeper:3.4.14 - restart: always - hostname: 'zoo1' - ports: - - 2181:2181 - environment: - ZOO_MY_ID: 1 - ZOO_SERVERS: server.1=0.0.0.0:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888 - - zoo2: - image: zookeeper:3.4.14 - restart: always - hostname: 'zoo2' - ports: - - 2182:2181 - environment: - ZOO_MY_ID: 2 - ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=0.0.0.0:2888:3888 server.3=zoo3:2888:3888 - - zoo3: - image: zookeeper:3.4.14 - restart: always - hostname: 'zoo3' - ports: - - 2183:2181 - environment: - ZOO_MY_ID: 3 - ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=0.0.0.0:2888:3888 diff --git a/packages/caliper-cli/lib/zooservice/startZooServiceCommand.js b/packages/caliper-cli/lib/zooservice/startZooServiceCommand.js deleted file mode 100644 index 15cd85680..000000000 --- a/packages/caliper-cli/lib/zooservice/startZooServiceCommand.js +++ /dev/null @@ -1,47 +0,0 @@ -/* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -'use strict'; - -const Start = require ('./lib/startZooService'); - -// enforces singletons -const checkFn = (argv, options) => { - - ['caliper-zooconfig'].forEach((e)=>{ - if (Array.isArray(argv[e])){ - throw new Error(`Option ${e} can only be specified once`); - } - }); - - return true; -}; -module.exports._checkFn = checkFn; -module.exports.command = 'start'; -module.exports.describe = 'Start a zookeeper service'; -module.exports.builder = function (yargs){ - yargs.options({ - 'caliper-zooconfig' : {describe: 'Path to a zookeeper service yaml file.', type: 'string' } - }); - yargs.usage('caliper zooservice start --caliper-zooconfig ./my-zoo-service.yaml'); - - // enforce singletons - yargs.check(checkFn); - - return yargs; -}; - -module.exports.handler = (argv) => { - return argv.thePromise = Start.handler(argv); -}; diff --git a/packages/caliper-cli/lib/zooservice/stopZooServiceCommand.js b/packages/caliper-cli/lib/zooservice/stopZooServiceCommand.js deleted file mode 100644 index a05ad94a0..000000000 --- a/packages/caliper-cli/lib/zooservice/stopZooServiceCommand.js +++ /dev/null @@ -1,47 +0,0 @@ -/* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -'use strict'; - -const Stop = require ('./lib/stopZooService'); - -// enforces singletons -const checkFn = (argv, options) => { - - ['caliper-zooconfig'].forEach((e)=>{ - if (Array.isArray(argv[e])){ - throw new Error(`Option ${e} can only be specified once`); - } - }); - - return true; -}; -module.exports._checkFn = checkFn; -module.exports.command = 'stop'; -module.exports.describe = 'Stop a zookeeper service'; -module.exports.builder = function (yargs){ - yargs.options({ - 'caliper-zooconfig' : {describe: 'Path to a zookeeper service yaml file.', type: 'string' } - }); - yargs.usage('caliper zooservice stop --caliper-zooconfig ./my-zoo-service.yaml'); - - // enforce singletons - yargs.check(checkFn); - - return yargs; -}; - -module.exports.handler = (argv) => { - return argv.thePromise = Stop.handler(argv); -}; diff --git a/packages/caliper-cli/package.json b/packages/caliper-cli/package.json index 1be252de4..e8bc99632 100644 --- a/packages/caliper-cli/package.json +++ b/packages/caliper-cli/package.json @@ -1,5 +1,6 @@ { "name": "@hyperledger/caliper-cli", + "description": "Hyperledger Caliper CLI, for convenience running of a performance benchmark to test blockchain technologies", "version": "0.1.0", "repository": { "type": "git", diff --git a/packages/caliper-core/index.js b/packages/caliper-core/index.js index 41da59518..abf8b61ae 100644 --- a/packages/caliper-core/index.js +++ b/packages/caliper-core/index.js @@ -16,7 +16,6 @@ module.exports.BlockchainInterface = require('./lib/blockchain-interface'); module.exports.CaliperLocalClient = require('./lib/client/caliper-local-client'); -module.exports.CaliperZooClient = require('./lib/client/caliper-zoo-client'); module.exports.TxStatus = require('./lib/transaction-status'); module.exports.CaliperUtils = require('./lib/utils/caliper-utils'); module.exports.Version = require('./lib/utils/version'); diff --git a/packages/caliper-core/lib/blockchain.js b/packages/caliper-core/lib/blockchain.js index 057c7996d..7f855dd59 100644 --- a/packages/caliper-core/lib/blockchain.js +++ b/packages/caliper-core/lib/blockchain.js @@ -14,6 +14,8 @@ 'use strict'; +const Logger = require('./utils/caliper-utils').getLogger('blockchain'); + /** * BlockChain class, define operations to interact with the blockchain system under test */ @@ -28,10 +30,10 @@ class Blockchain { } /** - * return the blockchain's type + * return the blockchain type * @return {string} type of the blockchain */ - gettype() { + getType() { return this.bcType; } @@ -54,7 +56,7 @@ class Blockchain { } /** - * Install smart contract(s), detail informations are defined in the blockchain configuration file + * Install smart contract(s), detail information is defined in the blockchain configuration file * @async */ async installSmartContract() { @@ -86,7 +88,7 @@ class Blockchain { /** * Invoke smart contract/submit transactions and return corresponding transactions' status * @param {Object} context context object - * @param {String} contractID identiy of the contract + * @param {String} contractID identity of the contract * @param {String} contractVer version of the contract * @param {Array} args array of JSON formatted arguments for multiple transactions * @param {Number} timeout request timeout, in second @@ -117,7 +119,7 @@ class Blockchain { /** * Query state from the ledger using a smart contract * @param {Object} context context object - * @param {String} contractID identiy of the contract + * @param {String} contractID identity of the contract * @param {String} contractVer version of the contract * @param {Array} args array of JSON formatted arguments * @param {Number} timeout request timeout, in seconds @@ -148,11 +150,11 @@ class Blockchain { /** * Query state from the ledger * @param {Object} context context object from getContext - * @param {String} contractID identiy of the contract + * @param {String} contractID identity of the contract * @param {String} contractVer version of the contract * @param {String} key lookup key * @param {String=} [fcn] query function name - * @return {Object} as invokeSmateContract() + * @return {Object} as invokeSmartContract() */ async queryState(context, contractID, contractVer, key, fcn) { return await this.bcObj.queryState(context, contractID, contractVer, key, fcn); @@ -160,11 +162,11 @@ class Blockchain { /** * Calculate the default transaction statistics - * @param {Array} results array of txStatus + * @param {Array} resultArray array of txStatus * @param {Boolean} detail indicates whether to keep detailed information * @return {JSON} txStatistics JSON object */ - getDefaultTxStats(results, detail) { + getDefaultTxStats(resultArray, detail) { let succ = 0, fail = 0, delay = 0; let minFinal, maxFinal, minCreate, maxCreate; let maxLastFinal; @@ -173,8 +175,8 @@ class Blockchain { let sTPTotal = 0; let sTTotal = 0; let invokeTotal = 0; - for(let i = 0 ; i < results.length ; i++) { - let stat = results[i]; + for(let i = 0 ; i < resultArray.length ; i++) { + let stat = resultArray[i]; sTPTotal = sTPTotal + stat.Get('sTP'); sTTotal = sTTotal + stat.Get('sT'); invokeTotal += stat.Get('invokeLatency'); @@ -247,7 +249,7 @@ class Blockchain { 'sTPTotal': sTPTotal, 'sTTotal': sTTotal, 'invokeTotal': invokeTotal, - 'length': results.length + 'length': resultArray.length }; return stats; } @@ -255,15 +257,15 @@ class Blockchain { /** * merge an array of default 'txStatistics', the result is in first object of the array * Note even failed the first object of the array may still be changed - * @param {Array} results txStatistics array + * @param {Array} resultArray txStatistics array * @return {Number} 0 if failed; otherwise 1 */ - static mergeDefaultTxStats(results) { + static mergeDefaultTxStats(resultArray) { try{ // skip invalid result let skip = 0; - for(let i = 0 ; i < results.length ; i++) { - let result = results[i]; + for(let i = 0 ; i < resultArray.length ; i++) { + let result = resultArray[i]; if(!result.hasOwnProperty('succ') || !result.hasOwnProperty('fail') || (result.succ + result.fail) === 0) { skip++; @@ -274,16 +276,16 @@ class Blockchain { } if(skip > 0) { - results.splice(0, skip); + resultArray.splice(0, skip); } - if(results.length === 0) { + if(resultArray.length === 0) { return 0; } - let r = results[0]; - for(let i = 1 ; i < results.length ; i++) { - let v = results[i]; + let r = resultArray[0]; + for(let i = 1 ; i < resultArray.length ; i++) { + let v = resultArray[i]; if(!v.hasOwnProperty('succ') || !v.hasOwnProperty('fail') || (v.succ + v.fail) === 0) { continue; } @@ -323,7 +325,7 @@ class Blockchain { return 1; } catch(err) { - //throw err; + Logger.error(err); return 0; } } diff --git a/packages/caliper-core/lib/caliper-flow.js b/packages/caliper-core/lib/caliper-flow.js index 83bc873db..5cfe91f81 100644 --- a/packages/caliper-core/lib/caliper-flow.js +++ b/packages/caliper-core/lib/caliper-flow.js @@ -17,14 +17,13 @@ const Blockchain = require('./blockchain'); const CaliperUtils = require('./utils/caliper-utils'); const ClientOrchestrator = require('./client/client-orchestrator'); - -const Monitor = require('./monitor/monitor'); +const MonitorOrchestrator = require('./monitor/monitor-orchestrator'); const Report = require('./report/report'); -const Test = require('./test/defaultTest'); +const DefaultTest = require('./test-runners/default-test'); +const TestObserver = require('./test-observers/test-observer'); +const BenchValidator = require('./utils/benchmark-validator'); -const demo = require('./gui/src/demo.js'); const path = require('path'); - const logger = CaliperUtils.getLogger('caliper-flow'); /** @@ -44,17 +43,24 @@ module.exports.run = async function(absConfigFile, absNetworkFile, admin, client // Retrieve flow conditioning options const flowOpts = CaliperUtils.getFlowOptions(); + let configObject = CaliperUtils.parseYaml(absConfigFile); + let networkObject = CaliperUtils.parseYaml(absNetworkFile); + + // Validate configObject (benchmark configuration file) + BenchValidator.validateObject(configObject); logger.info('####### Caliper Test #######'); const adminClient = new Blockchain(admin); const clientOrchestrator = new ClientOrchestrator(absConfigFile); - const monitor = new Monitor(absConfigFile); - const report = new Report(monitor); - report.createReport(absConfigFile, absNetworkFile, adminClient.gettype()); - demo.init(); + const monitorOrchestrator = new MonitorOrchestrator(absConfigFile); - let configObject = CaliperUtils.parseYaml(absConfigFile); - let networkObject = CaliperUtils.parseYaml(absNetworkFile); + // Test observer is dynamically loaded, but defaults to none + const observerType = (configObject.observer && configObject.observer.type) ? configObject.observer.type : 'none'; + const testObserver = new TestObserver(observerType, absConfigFile); + + // Report + const report = new Report(monitorOrchestrator); + report.createReport(absConfigFile, absNetworkFile, adminClient.getType()); try { // Conditional running of 'start' commands @@ -71,7 +77,7 @@ module.exports.run = async function(absConfigFile, absNetworkFile, admin, client } } - // Conditional network initialisation + // Conditional network initialization if (!flowOpts.performInit) { logger.info('Skipping initialization phase due to benchmark flow conditioning'); } else { @@ -89,19 +95,19 @@ module.exports.run = async function(absConfigFile, absNetworkFile, admin, client if (!flowOpts.performTest) { logger.info('Skipping benchmark test phase due to benchmark flow conditioning'); } else { - // Start the monitors + // Start all the monitors try { - await monitor.start(); - logger.info('Started monitor successfully'); + await monitorOrchestrator.startAllMonitors(); + logger.info('Started monitors successfully'); } catch (err) { - logger.error('Could not start monitor, ' + (err.stack ? err.stack : err)); + logger.error('Could not start monitors, ' + (err.stack ? err.stack : err)); } let testIdx = 0; let numberOfClients = await clientOrchestrator.init(); let clientArgs = await adminClient.prepareClients(numberOfClients); - const tester = new Test(clientArgs, absNetworkFile, clientOrchestrator, clientFactory, workspace, report, demo, monitor); + const tester = new DefaultTest(clientArgs, absNetworkFile, clientOrchestrator, clientFactory, workspace, report, testObserver, monitorOrchestrator); const allTests = configObject.test.rounds; for (let test of allTests) { ++testIdx; @@ -112,8 +118,7 @@ module.exports.run = async function(absConfigFile, absNetworkFile, admin, client logger.info('---------- Finished Test ----------\n'); report.printResultsByRound(); - monitor.printMaxStats(); - await monitor.stop(); + await monitorOrchestrator.stopAllMonitors(); const date = new Date().toISOString().replace(/-/g,'').replace(/:/g,'').substr(0,15); const outFile = path.join(process.cwd(), `report-${date}.html`); @@ -121,8 +126,6 @@ module.exports.run = async function(absConfigFile, absNetworkFile, admin, client clientOrchestrator.stop(); - demo.stopWatch(); - // NOTE: keep the below multi-line formatting intact, otherwise the indents will interfere with the template literal let testSummary = `# Test summary: ${successes} succeeded, ${failures} failed #`; logger.info(` @@ -136,6 +139,8 @@ ${'#'.repeat(testSummary.length)} logger.error(`Error: ${err.stack ? err.stack : err}`); errorStatus = 1; } finally { + await testObserver.stopWatch(); + // Conditional running of 'end' commands if (flowOpts.performEnd) { if (networkObject.hasOwnProperty('caliper') && networkObject.caliper.hasOwnProperty('command') && networkObject.caliper.command.hasOwnProperty('end')) { diff --git a/packages/caliper-core/lib/client/caliper-local-client.js b/packages/caliper-core/lib/client/caliper-local-client.js index 1765020ae..186bc98aa 100644 --- a/packages/caliper-core/lib/client/caliper-local-client.js +++ b/packages/caliper-core/lib/client/caliper-local-client.js @@ -14,11 +14,12 @@ 'use strict'; -const cfUtil = require('../config/config-util.js'); +const Config = require('../config/config-util.js'); const CaliperUtils = require('../utils/caliper-utils.js'); const Logger = CaliperUtils.getLogger('local-client.js'); const bc = require('../blockchain.js'); const RateControl = require('../rate-control/rateControl.js'); +const PrometheusClient = require('../prometheus/prometheus-push-client'); /** * Class for Client Interaction @@ -38,29 +39,57 @@ class CaliperLocalClient { this.trimType = 0; this.trim = 0; this.startTime = 0; + + // Prometheus related + this.prometheusClient = new PrometheusClient(); + this.totalTxCount = 0; + this.totalTxDelay = 0; } /** - * Calculate realtime transaction statistics and send the txUpdated message + * Calculate real-time transaction statistics and send the txUpdated message */ txUpdate() { let newNum = this.txNum - this.txLastNum; this.txLastNum += newNum; + // get a copy to work from let newResults = this.results.slice(0); this.results = []; - if(newResults.length === 0 && newNum === 0) { + if (newResults.length === 0 && newNum === 0) { return; } - let newStats; - if(newResults.length === 0) { + let publish = true; + if (newResults.length === 0) { newStats = bc.createNullDefaultTxStats(); - } - else { + publish = false; // no point publishing nothing!! + } else { newStats = this.blockchain.getDefaultTxStats(newResults, false); } - process.send({type: 'txUpdated', data: {submitted: newNum, committed: newStats}}); + + // Update monitor + if (this.prometheusClient.gatewaySet() && publish){ + // Send to Prometheus push gateway + + // TPS and latency batch results for this current txUpdate limited set + const batchTxCount = newStats.succ + newStats.fail; + const batchTPS = (batchTxCount/this.txUpdateTime)*1000; // txUpdate is in ms + const batchLatency = newStats.delay.sum/batchTxCount; + this.prometheusClient.push('caliper_tps', batchTPS); + this.prometheusClient.push('caliper_latency', batchLatency); + this.prometheusClient.push('caliper_txn_submit_rate', (newNum/this.txUpdateTime)*1000); // txUpdate is in ms + + // Numbers for test round only + this.totalTxnSuccess += newStats.succ; + this.totalTxnFailure += newStats.fail; + this.prometheusClient.push('caliper_txn_success', this.totalTxnSuccess); + this.prometheusClient.push('caliper_txn_failure', this.totalTxnFailure); + this.prometheusClient.push('caliper_txn_pending', (this.txNum - (this.totalTxnSuccess + this.totalTxnFailure))); + } else { + // client-orchestrator based update + process.send({type: 'txUpdated', data: {type: 'txUpdate', submitted: newNum, committed: newStats}}); + } if (this.resultStats.length === 0) { switch (this.trimType) { @@ -89,17 +118,38 @@ class CaliperLocalClient { } } + /** + * Method to reset values between `init` and `test` phase + */ + txReset(){ + + // Reset txn counters + this.txNum = 0; + this.txLastNum = 0; + + if (this.prometheusClient.gatewaySet()) { + // Reset Prometheus + this.totalTxnSuccess = 0; + this.totalTxnFailure = 0; + this.prometheusClient.push('caliper_txn_success', 0); + this.prometheusClient.push('caliper_txn_failure', 0); + this.prometheusClient.push('caliper_txn_pending', 0); + } else { + // Reset Local + process.send({type: 'txReset', data: {type: 'txReset'}}); + } + } + /** * Add new test result into global array * @param {Object} result test result, should be an array or a single JSON object */ addResult(result) { - if(Array.isArray(result)) { // contain multiple results + if (Array.isArray(result)) { // contain multiple results for(let i = 0 ; i < result.length ; i++) { this.results.push(result[i]); } - } - else { + } else { this.results.push(result); } } @@ -115,6 +165,7 @@ class CaliperLocalClient { this.txNum = 0; this.txLastNum = 0; + // TODO: once prometheus is enabled, trim occurs as part of the retrieval query // conditionally trim beginning and end results for this test run if (msg.trim) { if (msg.txDuration) { @@ -126,6 +177,20 @@ class CaliperLocalClient { } else { this.trimType = 0; } + + // Prometheus is specified if msg.pushUrl !== null + if (msg.pushUrl !== null) { + // - ensure counters reset + this.totalTxnSubmitted = 0; + this.totalTxnSuccess = 0; + this.totalTxnFailure = 0; + // - Ensure gateway base URL is set + if (!this.prometheusClient.gatewaySet()){ + this.prometheusClient.setGateway(msg.pushUrl); + } + // - set target for this round test/round/client + this.prometheusClient.configureTarget(msg.label, msg.testRound, msg.clientIdx); + } } /** @@ -138,7 +203,7 @@ class CaliperLocalClient { /** * Put a task to immediate queue of NodeJS event loop - * @param {function} func The function needed to be exectued immediately + * @param {function} func The function needed to be executed immediately * @return {Promise} Promise of execution */ setImmediatePromise(func) { @@ -152,21 +217,17 @@ class CaliperLocalClient { /** * Perform test with specified number of transactions - * @param {JSON} msg start test message * @param {Object} cb callback module - * @param {Object} context blockchain context - * @return {Promise} promise object + * @param {Object} number number of transactions to submit + * @param {Object} rateController rate controller object + * @async */ - async runFixedNumber(msg, cb, context) { + async runFixedNumber(cb, number, rateController) { Logger.info('Info: client ' + process.pid + ' start test runFixedNumber()' + (cb.info ? (':' + cb.info) : '')); - let rateControl = new RateControl(msg.rateControl, msg.clientIdx, msg.roundIdx); - await rateControl.init(msg); - - await cb.init(this.blockchain, context, msg.args); this.startTime = Date.now(); let promises = []; - while(this.txNum < msg.numb) { + while(this.txNum < number) { // If this function calls cb.run() too quickly, micro task queue will be filled with unexecuted promises, // and I/O task(s) will get no chance to be execute and fall into starvation, for more detail info please visit: // https://snyk.io/blog/nodejs-how-even-quick-async-functions-can-block-the-event-loop-starve-io/ @@ -176,28 +237,22 @@ class CaliperLocalClient { return Promise.resolve(); })); }); - await rateControl.applyRateControl(this.startTime, this.txNum, this.results, this.resultStats); + await rateController.applyRateControl(this.startTime, this.txNum, this.results, this.resultStats); } await Promise.all(promises); - await rateControl.end(); - return await this.blockchain.releaseContext(context); + this.endTime = Date.now(); } /** * Perform test with specified test duration - * @param {JSON} msg start test message * @param {Object} cb callback module - * @param {Object} context blockchain context - * @return {Promise} promise object + * @param {Object} duration duration to run for + * @param {Object} rateController rate controller object + * @async */ - async runDuration(msg, cb, context) { + async runDuration(cb, duration, rateController) { Logger.info('Info: client ' + process.pid + ' start test runDuration()' + (cb.info ? (':' + cb.info) : '')); - let rateControl = new RateControl(msg.rateControl, msg.clientIdx, msg.roundIdx); - await rateControl.init(msg); - const duration = msg.txDuration; // duration in seconds - - await cb.init(this.blockchain, context, msg.args); this.startTime = Date.now(); let promises = []; @@ -211,12 +266,11 @@ class CaliperLocalClient { return Promise.resolve(); })); }); - await rateControl.applyRateControl(this.startTime, this.txNum, this.results, this.resultStats); + await rateController.applyRateControl(this.startTime, this.txNum, this.results, this.resultStats); } await Promise.all(promises); - await rateControl.end(); - return await this.blockchain.releaseContext(context); + this.endTime = Date.now(); } /** @@ -225,7 +279,7 @@ class CaliperLocalClient { */ clearUpdateInter(txUpdateInter) { // stop reporter - if(txUpdateInter) { + if (txUpdateInter) { clearInterval(txUpdateInter); txUpdateInter = null; this.txUpdate(); @@ -234,49 +288,87 @@ class CaliperLocalClient { /** * Perform the test - * @param {JSON} msg start test message + * @param {JSON} test start test message + * message = { + * type: 'test', + * label : label name, + * numb: total number of simulated txs, + * rateControl: rate controller to use + * trim: trim options + * args: user defined arguments, + * cb : path of the callback js file, + * config: path of the blockchain config file + * clientIdx = this client index, + * clientArgs = clientArgs[clientIdx], + * totalClients = total number of clients, + * pushUrl = the url for the push gateway + * }; * @return {Promise} promise object */ - async doTest(msg) { - Logger.debug('doTest() with:', msg); - let cb = require(CaliperUtils.resolvePath(msg.cb, msg.root)); + async doTest(test) { + Logger.debug('doTest() with:', test); + let cb = require(CaliperUtils.resolvePath(test.cb, test.root)); - this.beforeTest(msg); + this.beforeTest(test); - let txUpdateTime = cfUtil.get(cfUtil.keys.TxUpdateTime, 1000); - Logger.info('txUpdateTime: ' + txUpdateTime); + this.txUpdateTime = Config.get(Config.keys.TxUpdateTime, 1000); + Logger.info('txUpdateTime: ' + this.txUpdateTime); const self = this; - let txUpdateInter = setInterval( () => { self.txUpdate(); } , txUpdateTime); + let txUpdateInter = setInterval( () => { self.txUpdate(); } , self.txUpdateTime); try { - let context = await this.blockchain.getContext(msg.label, msg.clientargs, msg.clientIdx, msg.txFile); - if(typeof context === 'undefined') { + let context = await this.blockchain.getContext(test.label, test.clientArgs, test.clientIdx, test.txFile); + if (typeof context === 'undefined') { context = { engine : { submitCallback : (count) => { self.submitCallback(count); } } }; - } - else { + } else { context.engine = { submitCallback : (count) => { self.submitCallback(count); } }; } - if (msg.txDuration) { - await this.runDuration(msg, cb, context); + // Configure + let rateController = new RateControl(test.rateControl, test.clientIdx, test.testRound); + await rateController.init(test); + + // Run init phase of callback + Logger.info(`Info: client ${process.pid} init test ${(cb.info ? (':' + cb.info) : 'phase')}`); + await cb.init(this.blockchain, context, test.args); + // Reset and sleep the configured update duration to flush any results that are resulting from the 'init' stage + this.txReset(); + await CaliperUtils.sleep(this.txUpdateTime); + + // Run the test loop + if (test.txDuration) { + const duration = test.txDuration; // duration in seconds + await this.runDuration(cb, duration, rateController); } else { - await this.runFixedNumber(msg, cb, context); + const number = test.numb; + await this.runFixedNumber(cb, number, rateController); } + // Clean up + await rateController.end(); + await this.blockchain.releaseContext(context); this.clearUpdateInter(txUpdateInter); await cb.end(); + // Return the results and time stamps if (this.resultStats.length > 0) { - return this.resultStats[0]; - } - else { - return this.blockchain.createNullDefaultTxStats(); + return { + results: this.resultStats[0], + start: this.startTime, + end: this.endTime + }; + } else { + return { + results: this.blockchain.createNullDefaultTxStats(), + start: this.startTime, + end: this.endTime + }; } } catch (err) { this.clearUpdateInter(); diff --git a/packages/caliper-core/lib/client/caliper-zoo-client.js b/packages/caliper-core/lib/client/caliper-zoo-client.js deleted file mode 100644 index 663ed8618..000000000 --- a/packages/caliper-core/lib/client/caliper-zoo-client.js +++ /dev/null @@ -1,290 +0,0 @@ -/* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -'use strict'; - -const Util = require('../utils/caliper-utils'); -const logger = Util.getLogger('zoo-client.js'); -const Blockchain = require('../blockchain.js'); -const ZooKeeper = require('node-zookeeper-client'); -const zkUtil = require('./zoo-util.js'); -const clientUtil = require('./client-util.js'); -const path = require('path'); - - -/** - * Class for ZooKeeper Clients - * zookeeper structure - * /caliper---clients---client_xxx // list of clients - * | |--client_yyy - * | |--.... - * |--client_xxx_in---msg_xxx {message} - * | |--msg_xxx {message} - * | |--...... - * |--client_xxx_out---msg_xxx {message} - * | |--msg_xxx {message} - * |--client_yyy_in---... - */ -class CaliperZooClient { - - /** - * Create the zoo client - * @param {Object} address the zookeeper address - * @param {Object} clientFactory blockchain client factory - * @param {String} networkRoot fully qualified path to the root location of network files - */ - constructor(address, clientFactory, networkRoot) { - this.address = address; - this.clientFactory = clientFactory; - this.zk = ZooKeeper.createClient(address); - this.clientID = ''; - this.inNode = ''; - this.outNode = ''; - this.closed = false; - this.results = []; // contains testResult message data - this.updates = []; // contains txUpdated message data - this.updateTail = 0; - this.updateInter = null; - this.updateTime = 1000; - this.networkRoot = networkRoot; - } - - /** - * Remove unused znodes - * @return {Promise} promise object - */ - clear() { - let promises = []; - if(this.inNode !== '') { - promises.push(zkUtil.removeChildrenP(this.zk, this.inNode, 'Failed to remove children due to')); - } - if(this.outNode !== '') { - promises.push(zkUtil.removeChildrenP(this.zk, this.outNode, 'Failed to remove children due to')); - } - clientUtil.stop(); - return Promise.all(promises); - } - - /** - * Close zk client - */ - close() { - logger.info('closing zookeeper client...'); - if (this.closed) { - return; - } - this.closed = true; - this.clear().then(()=>{ - let promises = []; - if(this.inNode !== '') { - promises.push(zkUtil.removeP(this.zk, this.inNode, -1, 'Failed to remove inNode due to')); - } - if(this.outNode !== '') { - promises.push(zkUtil.removeP(this.zk, this.outNode, -1, 'Failed to remove inNode due to')); - } - }).then(()=>{ - logger.info('Node ' + this.inNode + ' ' + this.outNode + ' is deleted'); - this.inNode = ''; - this.outNode = ''; - this.zk.close(); - }).catch((err)=>{ - this.inNode = ''; - this.outNode = ''; - this.zk.close(); - }); - } - - /** - * Write data (send message) into zk - * @param {Buffer} data message data - * @return {Promise} promise object - */ - write(data) { - return zkUtil.createP(this.zk, this.outNode+'/msg_', data, ZooKeeper.CreateMode.EPHEMERAL_SEQUENTIAL, 'Failed to send message (create node) due to'); - } - - /** - * Generate and send txUpdated message - */ - txUpdate() { - let len = this.updates.length; - if(len === this.updateTail) { - return; - } - - let submitted = 0; - let committed = []; - for(let i = this.updateTail ; i < len ; i++) { - submitted += this.updates[i].submitted; - committed.push(this.updates[i].committed); - } - this.updateTail = len; - - let message = {type: 'txUpdated', data: {submitted: submitted}}; - if(Blockchain.mergeDefaultTxStats(committed) === 0) { - message.data.committed = Blockchain.createNullDefaultTxStats(); - } - else { - message.data.committed = committed[0]; - } - let buf = new Buffer(JSON.stringify(message)); - this.write(buf); - } - - /** - * Initialise global variables before test - */ - beforeTest() { - this.results = []; - this.updates = []; - this.updateTail = 0; - const self = this; - this.updateInter = setInterval( () => { self.txUpdate(); } , self.updateTime); - } - - /** - * Send results and release resources after test - * @return {Promise} promise object - */ - afterTest() { - if(this.updateInter) { - clearInterval(this.updateInter); - this.updateInter = null; - this.txUpdate(); - } - - return Util.sleep(200).then(()=>{ - let message = {type: 'testResult', data: this.results[0]}; - if(Blockchain.mergeDefaultTxStats(this.results) === 0) { - message = {type: 'testResult', data: Blockchain.createNullDefaultTxStats()}; - } - else { - message = {type: 'testResult', data: this.results[0]}; - } - let buf = new Buffer(JSON.stringify(message)); - return this.write(buf); - }).catch((err) => { - logger.error(err); - return Promise.resolve(); - }); - } - - /** - * Message handler - * @param {Object} data message received - * @return {Promise} returned bool value which indicates whether the zk connection has been closed or not - */ - zooMessageCallback(data) { - let msg = JSON.parse(data.toString()); - logger.info('Receive message, type='+msg.type); - - switch(msg.type) { - case 'test': { - this.beforeTest(); - msg.root = this.networkRoot; - zkUtil.removeChildrenP(this.zk, this.outNode, 'Failed to remove children in outNode due to').then(()=>{ - return clientUtil.startTest(msg.clients, msg, msg.clientargs, this.updates, this.results, this.clientFactory); - }).then(() => { - return this.afterTest(); - }).catch((err)=>{ - logger.error('==Exception while testing, ' + err); - return this.afterTest(); - }); - break; - } - case 'quit': { - this.clear(); - break; - } - default: { - clientUtil.sendMessage(msg); - break; - } - } - return Promise.resolve(this.closed); - } - - /** - * Waiting for messages by watching corresponding zk nodes - * @return {Promise} promise object - */ - watch() { - return zkUtil.watchMsgQueueP( - this.zk, - this.inNode, - (data) => { - return this.zooMessageCallback(data).catch((err) => { - logger.error('Exception encountered when watching message from zookeeper, due to:' + err); - return Promise.resolve(true); - }); - }, - 'Failed to watch children nodes in zookeeper' - ).catch((err) => { - logger.error(err); - return Promise.resolve(); - }); - } - - /** - * Start the zookeper client - */ - start(){ - const self = this; - this.zk.once('connected', function() { - logger.info('Connected to ZooKeeper'); - zkUtil.existsP(self.zk, zkUtil.NODE_ROOT, 'Failed to find NODE_ROOT due to').then((found)=>{ - if(found) { - return Promise.resolve(); - } - else { - return zkUtil.createP(self.zk, zkUtil.NODE_ROOT, null, ZooKeeper.CreateMode.PERSISTENT, 'Failed to create NODE_ROOT due to'); - } - }).then(()=>{ - return zkUtil.existsP(self.zk, zkUtil.NODE_CLIENT, 'Failed to find clients node due to'); - }).then((found)=>{ - if(found) { - return Promise.resolve(); - } - else { - return zkUtil.createP(self.zk, zkUtil.NODE_CLIENT, null, ZooKeeper.CreateMode.PERSISTENT, 'Failed to create clients node due to'); - } - }).then(()=>{ // create client node - let random = new Date().getTime(); - let clientPath = zkUtil.NODE_CLIENT + '/client_'+random+'_'; - return zkUtil.createP(self.zk, clientPath, null, ZooKeeper.CreateMode.EPHEMERAL_SEQUENTIAL, 'Failed to create client node due to'); - }).then((clientPath)=>{ - logger.info('Created client node:'+clientPath); - self.clientID = path.basename(clientPath); - self.inNode = zkUtil.getInNode(self.clientID); - self.outNode = zkUtil.getOutNode(self.clientID); - return zkUtil.createP(self.zk, self.inNode, null, ZooKeeper.CreateMode.PERSISTENT, 'Failed to create receiving queue due to'); - }).then((inPath)=>{ - logger.info('Created receiving queue at:'+inPath); - return zkUtil.createP(self.zk, self.outNode, null, ZooKeeper.CreateMode.PERSISTENT, 'Failed to create sending queue due to'); - }).then((outPath)=>{ - logger.info('Created sending queue at:'+outPath); - logger.info('Waiting for messages at:'+self.inNode+'......'); - self.watch(); - }).catch((err)=> { - logger.error(err.stack ? err.stack : err); - self.close(); - }); - }); - - this.zk.connect(); - } - -} - -module.exports = CaliperZooClient; diff --git a/packages/caliper-core/lib/client/client-orchestrator.js b/packages/caliper-core/lib/client/client-orchestrator.js index b02541a23..c70291b0c 100644 --- a/packages/caliper-core/lib/client/client-orchestrator.js +++ b/packages/caliper-core/lib/client/client-orchestrator.js @@ -15,76 +15,9 @@ 'use strict'; -const CLIENT_LOCAL = 'local'; -const CLIENT_ZOO = 'zookeeper'; - -const zkUtil = require('./zoo-util.js'); -const ZooKeeper = require('node-zookeeper-client'); -const clientUtil = require('./client-util.js'); - const util = require('../utils/caliper-utils'); const logger = util.getLogger('client.js'); - -/** - * Callback function to handle messages received from zookeeper clients - * @param {Object} data message data - * @param {Array} updates array to save txUpdate results - * @param {Array} results array to save test results - * @return {Promise} boolean value that indicates whether the test of corresponding client has already stopped - */ -function zooMessageCallback(data, updates, results) { - let msg = JSON.parse(data.toString()); - let stop = false; - switch(msg.type) { - case 'testResult': - results.push(msg.data); - stop = true; // stop watching - break; - case 'error': - logger.error('Client encountered error, ' + msg.data); - stop = true; // stop watching - break; - case 'txUpdated': - updates.push(msg.data); - stop = false; - break; - default: - logger.warn('Unknown message type: ' + msg.type); - stop = false; - break; - } - return Promise.resolve(stop); -} - -/** - * Start watching zookeeper - * @param {JSON} zoo zookeeper service informations - * @param {Array} updates array to save txUpdate data - * @param {Array} results array to save test resultss - * @return {Promsise} promise object - */ -function zooStartWatch(zoo, updates, results) { - let promises = []; - let zk = zoo.zk; - zoo.hosts.forEach((host)=>{ - let path = host.outnode; - let p = zkUtil.watchMsgQueueP( - zk, - path, - (data)=>{ - return zooMessageCallback(data, updates, results).catch((err) => { - logger.error('Exception encountered when watching message from zookeeper, due to:' + err); - return Promise.resolve(true); - }); - }, - 'Failed to watch zookeeper children' - ); - promises.push(p); - }); - return Promise.all(promises); -} - /** * Class for Client Orchestration */ @@ -96,35 +29,21 @@ class ClientOrchestrator { constructor(config) { let conf = util.parseYaml(config); this.config = conf.test.clients; - this.results = []; // output of recent test round this.updates = {id:0, data:[]}; // contains txUpdated messages + this.processes = {}; } /** - * Initialise client object + * Initialize client object * @return {Promise} promise object */ async init() { - if(this.config.hasOwnProperty('type')) { - switch(this.config.type) { - case CLIENT_LOCAL: - this.type = CLIENT_LOCAL; - if(this.config.hasOwnProperty('number')) { - this.number = this.config.number; - } - else { - this.number = 1; - } - return this.number; - case CLIENT_ZOO: - return await this._initZoo(); - default: - throw new Error('Unknown client type, should be local or zookeeper'); - } - } - else { - throw new Error('Failed to find client type in config file'); + if(this.config.hasOwnProperty('number')) { + this.number = this.config.number; + } else { + this.number = 1; } + return this.number; } /** @@ -139,46 +58,117 @@ class ClientOrchestrator { * cb : path of the callback js file, * config: path of the blockchain config file // TODO: how to deal with the local config file when transfer it to a remote client (via zookeeper), as well as any local materials like cyrpto keys?? * }; - * @param {JSON} message start message + * @param {JSON} test test specification * @param {Array} clientArgs each element of the array contains arguments that should be passed to corresponding test client - * @param {Report} report the report being built - * @param {any} finishArgs arguments that should be passed to finishCB, the callback is invoke as finishCB(this.results, finshArgs) - * @param {Object} clientFactory a factor used to spawn test clients + * @param {Object} clientFactory a factory used to spawn test clients + * @returns {Object[]} the test results array * @async */ - async startTest(message, clientArgs, report, finishArgs, clientFactory) { - this.results = []; + async startTest(test, clientArgs, clientFactory) { this.updates.data = []; this.updates.id++; - switch(this.type) { - case CLIENT_LOCAL: - await this._startLocalTest(message, clientArgs, clientFactory); - break; - case CLIENT_ZOO: - await this._startZooTest(message, clientArgs); - break; - default: - throw new Error(`Unknown client type: ${this.type}`); + const results = []; + await this._startTest(this.number, test, clientArgs, this.updates.data, results, clientFactory); + const testOutput = this.formatResults(results); + return testOutput; + } + + /** + * Start a test + * @param {Number} number test clients' count + * @param {JSON} test test specification + * @param {Array} clientArgs each element contains specific arguments for a client + * @param {Array} updates array to save txUpdate results + * @param {Array} results array to save the test results + * @param {Object} clientFactory a factory to spawn test clients + * @async + */ + async _startTest(number, test, clientArgs, updates, results, clientFactory) { + + // Conditionally launch clients on the test round. Subsequent tests should re-use existing clients. + if (Object.keys(this.processes).length === 0) { + // launch clients + const readyPromises = []; + this.processes = {}; + for (let i = 0 ; i < number ; i++) { + this.launchClient(updates, results, clientFactory, readyPromises); + } + // wait for all clients to have initialized + logger.info(`Waiting for ${readyPromises.length} clients to be ready... `); + + await Promise.all(readyPromises); + + logger.info(`${readyPromises.length} clients ready, starting test phase`); + } else { + logger.info(`Existing ${Object.keys(this.processes).length} clients will be reused in next test round... `); + } + + let txPerClient; + let totalTx = test.numb; + if (test.numb) { + // Run specified number of transactions + txPerClient = Math.floor(test.numb / number); + + // trim should be based on client number if specified with txNumber + if (test.trim) { + test.trim = Math.floor(test.trim / number); + } + + if(txPerClient < 1) { + txPerClient = 1; + } + test.numb = txPerClient; + } else if (test.txDuration) { + // Run for time specified txDuration based on clients + // Do nothing, we run for the time specified within test.txDuration + } else { + throw new Error('Unconditioned transaction rate driving mode'); + } + + let promises = []; + let idx = 0; + for (let id in this.processes) { + let client = this.processes[id]; + let p = new Promise((resolve, reject) => { + client.obj.promise = { + resolve: resolve, + reject: reject + }; + }); + promises.push(p); + client.results = results; + client.updates = updates; + test.clientArgs = clientArgs[idx]; + test.clientIdx = idx; + test.totalClients = number; + + if(totalTx % number !== 0 && idx === number-1){ + test.numb = totalTx - txPerClient*(number - 1); + } + + // send test specification to client and update idx + client.obj.send(test); + idx++; } - await report.processResult(this.results, finishArgs); + await Promise.all(promises); + // clear promises + for (let client in this.processes) { + if (client.obj && client.ob.promise) { + delete client.obj.promise; + } + } } /** - * Stop the client + * Stop all test clients(child processes) */ stop() { - switch(this.type) { - case CLIENT_ZOO: - this._stopZoo(); - break; - case CLIENT_LOCAL: - clientUtil.stop(); - break; - default: - // nothing to do + for (let pid in this.processes) { + this.processes[pid].obj.kill(); } + this.processes = {}; } /** @@ -190,210 +180,150 @@ class ClientOrchestrator { } /** - * functions for CLIENT_LOCAL - */ + * Call the Promise function for a process + * @param {String} pid pid of the client process + * @param {Boolean} isResolve indicates resolve(true) or reject(false) + * @param {Object} msg input for the Promise function + * @param {Boolean} isReady indicates promise type ready(true) promise(false) + */ + setPromise(pid, isResolve, msg, isReady) { + const client = this.processes[pid]; + if (client) { + const type = isReady ? 'ready' : 'promise'; + const clientObj = client.obj; + if(clientObj && clientObj[type] && typeof clientObj[type] !== 'undefined') { + if(isResolve) { + clientObj[type].resolve(msg); + } + else { + clientObj[type].reject(msg); + } + } else { + throw new Error('Unconditioned case within setPromise()'); + } + } + } /** - * Start the test - * @param {JSON} message start messages - * @param {Array} clientArgs arguments for the test clients - * @param {Object} clientFactory a factory to spawn clients - * @return {Promise} promise object - * @async + * Push test result from a child process into the global array + * @param {String} pid pid of the child process + * @param {Object} data test result */ - async _startLocalTest(message, clientArgs, clientFactory) { - message.totalClients = this.number; - return await clientUtil.startTest(this.number, message, clientArgs, this.updates.data, this.results, clientFactory); + pushResult(pid, data) { + let p = this.processes[pid]; + if (p && p.results && typeof p.results !== 'undefined') { + p.results.push(data); + } } /** - * functions for CLIENT_ZOO - */ - /** - * zookeeper structure - * /caliper---clients---client_xxx // list of clients - * | |--client_yyy - * | |--.... - * |--client_xxx_in---msg_xxx {message} - * | |--msg_xxx {message} - * | |--...... - * |--client_xxx_out---msg_xxx {message} - * | |--msg_xxx {message} - * |--client_yyy_in---... - */ + * Push update value from a child process into the global array + * @param {String} pid pid of the child process + * @param {Object} data update value + */ + pushUpdate(pid, data) { + let p = this.processes[pid]; + if (p && p.updates && typeof p.updates !== 'undefined') { + p.updates.push(data); + } + } /** - * Connect to zookeeper server and look up available test clients - * @return {Promise} number of available test clients + * Launch a client process to do the test + * @param {Array} updates array to save txUpdate results + * @param {Array} results array to save the test results + * @param {Object} clientFactory a factory to spawn clients + * @param {Array} readyPromises array to hold ready promises */ - _initZoo() { - const TIMEOUT = 5000; - this.type = CLIENT_ZOO; - this.zoo = { - server: '', - zk: null, - hosts: [], // {id, innode, outnode} - clientsPerHost: 1 - }; + launchClient(updates, results, clientFactory, readyPromises) { + let client = clientFactory.spawnWorker(); + let pid = client.pid.toString(); - if(!this.config.hasOwnProperty('zoo')) { - return Promise.reject('Failed to find zoo property in config file'); - } + logger.info('Launching client with PID ', pid); + this.processes[pid] = {obj: client, results: results, updates: updates}; - let configZoo = this.config.zoo; - if(configZoo.hasOwnProperty('server')) { - this.zoo.server = configZoo.server; - } - else { - return Promise.reject(new Error('Failed to find zookeeper server address in config file')); - } - if(configZoo.hasOwnProperty('clientsPerHost')) { - this.zoo.clientsPerHost = configZoo.clientsPerHost; - } - - let zk = ZooKeeper.createClient(this.zoo.server, { - sessionTimeout: TIMEOUT, - spinDelay : 1000, - retries: 0 - }); - this.zoo.zk = zk; - let zoo = this.zoo; - let connectHandle = setTimeout(()=>{ - logger.error('Could not connect to ZooKeeper'); - Promise.reject('Could not connect to ZooKeeper'); - }, TIMEOUT+100); let p = new Promise((resolve, reject) => { - zk.once('connected', () => { - logger.info('Connected to ZooKeeper'); - clearTimeout(connectHandle); - zkUtil.existsP(zk, zkUtil.NODE_CLIENT, 'Failed to find clients due to').then((found)=>{ - if(!found) { - // since zoo-client(s) should create the node if it does not exist,no caliper node means no valid zoo-client now. - throw new Error('Could not found clients node in zookeeper'); - } - - return zkUtil.getChildrenP( - zk, - zkUtil.NODE_CLIENT, - null, - 'Failed to list clients due to'); - }).then((clients) => { - // TODO: not support add/remove zookeeper clients now - logger.info('get zookeeper clients:' + clients); - for (let i = 0 ; i < clients.length ; i++) { - let clientID = clients[i]; - zoo.hosts.push({ - id: clientID, - innode: zkUtil.getInNode(clientID), - outnode:zkUtil.getOutNode(clientID) - }); - } - resolve(clients.length * zoo.clientsPerHost); - }).catch((err)=>{ - zk.close(); - return reject(err); - }); - }); + client.ready = { + resolve: resolve, + reject: reject + }; }); - logger.info('Connecting to ZooKeeper......'); - zk.connect(); - return p; - } - /** - * Start test on zookeeper mode - * @param {JSON} message start message - * @param {Array} clientArgs arguments for test clients - * @return {Promise} promise object - */ - _startZooTest(message, clientArgs) { - let number = this.zoo.hosts.length; - if (message.numb) { - // Run specified number of transactions - message.numb = Math.floor(message.numb / number); - if(message.numb < 1) { - message.numb = 1; - } - // trim should be based on client number if specified with txNumber - if (message.trim) { - message.trim = Math.floor(message.trim / number); - } - } else if (message.txDuration) { - // Run each client for time specified txDuration - // do nothing - } else { - return Promise.reject(new Error('Unconditioned transaction rate driving mode')); - } + readyPromises.push(p); - message.clients = this.zoo.clientsPerHost; - message.totalClients = this.zoo.clientsPerHost * number; - return this._sendZooMessage(message, clientArgs).then((number)=>{ - if(number > 0) { - return zooStartWatch(this.zoo, this.updates.data, this.results); - } - else { - return Promise.reject(new Error('Failed to start the remote test')); + const self = this; + client.on('message', function(msg) { + if (msg.type === 'ready') { + logger.info('Client ready message received'); + self.setPromise(pid, true, null, true); + } else if (msg.type === 'txUpdated') { + self.pushUpdate(pid, msg.data); + } else if (msg.type === 'txReset') { + self.pushUpdate(pid, msg.data); + } else if (msg.type === 'testResult') { + self.pushResult(pid, msg.data); + self.setPromise(pid, true, null); + } else if (msg.type === 'error') { + self.setPromise(pid, false, new Error('Client encountered error:' + msg.data)); + } else { + self.setPromise(pid, false, new Error('Client returned unexpected message type:' + msg.type)); } - }).catch((err)=>{ - logger.error('Failed to start the remote test'); - return Promise.reject(err); + }); + + client.on('error', function() { + self.setPromise(pid, false, new Error('Client encountered unexpected error')); + }); + + client.on('exit', function(code, signal) { + logger.info(`Client exited with code ${code}`); + self.setPromise(pid, false, new Error('Client already exited')); }); } /** - * Send message to test clients via zookeeper service - * @param {JSON} message message to be sent - * @param {Array} clientArgs arguments for test clients - * @return {Number} actual number of sent messages + * Send message to all child processes + * @param {JSON} message message + * @return {Number} number of child processes */ - _sendZooMessage(message, clientArgs) { - let promises = []; - let succ = 0; - let argsSlice, msgBuffer; - if(Array.isArray(clientArgs)) { - argsSlice = clientArgs.length / this.zoo.hosts.length; - } - else { - msgBuffer = new Buffer(JSON.stringify(message)); + sendMessage(message) { + for (let pid in this.processes) { + this.processes[pid].obj.send(message); } - this.zoo.hosts.forEach((host, idx)=>{ - let data; - if(Array.isArray(clientArgs)) { - let msg = message; - msg.clientargs = clientArgs.slice(idx * argsSlice, idx * argsSlice+argsSlice); - data = new Buffer(JSON.stringify(msg)); - } - else { - data = msgBuffer; - } - - let p = zkUtil.createP(this.zoo.zk, host.innode+'/msg_', data, ZooKeeper.CreateMode.EPHEMERAL_SEQUENTIAL, 'Failed to send message (create node) due to').then((path)=>{ - succ++; - return Promise.resolve(); - }).catch((err)=>{ - return Promise.resolve(); - }); - promises.push(p); - }); - return Promise.all(promises).then(()=>{ - return Promise.resolve(succ); - }); + return this.processes.length; } /** - * Stop the client + * Format the final test results for subsequent consumption from [ {result: [], start: val, end: val}, {result: [], start: val, end: val}, {result: [], start: val, end: val}] + * to {results: [val, val], start: val, end: val} + * @param {JSON[]} results an Array of JSON objects + * @return {JSON} an appropriately formatted result */ - _stopZoo() { - if(this.zoo && this.zoo.zk) { - let msg = {type: 'quit'}; - this._sendZooMessage(msg).then(()=>{ - setTimeout(()=>{ - this.zoo.zk.close(); - this.zoo.hosts = []; - }, 1000); - }); + formatResults(results) { + + let resultArray = []; + let allStartedTime = null; + let allFinishedTime = null; + for (const clientResult of results){ + // Start building the array of all client results + resultArray = resultArray.concat(clientResult.results); + + // Track all started/complete times + if (!allStartedTime || clientResult.start > allStartedTime) { + allStartedTime = clientResult.start; + } + + if (!allFinishedTime || clientResult.end < allFinishedTime) { + allFinishedTime = clientResult.end; + } } + + return { + results: resultArray, + start: allStartedTime, + end: allFinishedTime + }; } + } module.exports = ClientOrchestrator; diff --git a/packages/caliper-core/lib/client/client-util.js b/packages/caliper-core/lib/client/client-util.js deleted file mode 100644 index 693dfe9ac..000000000 --- a/packages/caliper-core/lib/client/client-util.js +++ /dev/null @@ -1,231 +0,0 @@ -/* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - - -'use strict'; -const logger = require('../utils/caliper-utils').getLogger('client-util.js'); -let processes = {}; // {pid:{obj, promise}} - -/** - * Call the Promise function for a process - * @param {String} pid pid of the client process - * @param {Boolean} isResolve indicates resolve(true) or reject(false) - * @param {Object} msg input for the Promise function - * @param {Boolean} isReady indicates promise type ready(true) promise(false) - */ -function setPromise(pid, isResolve, msg, isReady) { - const client = processes[pid]; - if (client) { - const type = isReady ? 'ready' : 'promise'; - const clientObj = client.obj; - if(clientObj && clientObj[type] && typeof clientObj[type] !== 'undefined') { - if(isResolve) { - clientObj[type].resolve(msg); - } - else { - clientObj[type].reject(msg); - } - } else { - throw new Error('Unconditioned case within setPromise()'); - } - } -} - -/** - * Push test result from a child process into the global array - * @param {String} pid pid of the child process - * @param {Object} data test result - */ -function pushResult(pid, data) { - let p = processes[pid]; - if(p && p.results && typeof p.results !== 'undefined') { - p.results.push(data); - } -} - -/** - * Push update value from a child process into the global array - * @param {String} pid pid of the child process - * @param {Object} data update value - */ -function pushUpdate(pid, data) { - let p = processes[pid]; - if(p && p.updates && typeof p.updates !== 'undefined') { - p.updates.push(data); - } -} - -/** - * Launch a client process to do the test - * @param {Array} updates array to save txUpdate results - * @param {Array} results array to save the test results - * @param {Object} clientFactory a factory to spawn clients - * @param {Array} readyPromises array to hold ready promises - */ -function launchClient(updates, results, clientFactory, readyPromises) { - let client = clientFactory.spawnWorker(); - let pid = client.pid.toString(); - - logger.info('Launching client with PID ', pid); - processes[pid] = {obj: client, results: results, updates: updates}; - - let p = new Promise((resolve, reject) => { - client.ready = { - resolve: resolve, - reject: reject - }; - }); - - readyPromises.push(p); - - client.on('message', function(msg) { - if(msg.type === 'ready') { - logger.info('Client ready message recieved'); - setPromise(pid, true, null, true); - } - else if(msg.type === 'testResult') { - pushResult(pid, msg.data); - setPromise(pid, true, null); - } - else if(msg.type === 'error') { - setPromise(pid, false, new Error('Client encountered error:' + msg.data)); - } - else if(msg.type === 'txUpdated') { - pushUpdate(pid, msg.data); - } - }); - - client.on('error', function(){ - setPromise(pid, false, new Error('Client encountered unexpected error')); - }); - - client.on('exit', function(code, signal){ - logger.info(`Client exited with code ${code}`); - setPromise(pid, false, new Error('Client already exited')); - }); -} - -/** - * Start a test - * @param {Number} number test clients' count - * @param {JSON} message start message - * @param {Array} clientArgs each element contains specific arguments for a client - * @param {Array} updates array to save txUpdate results - * @param {Array} results array to save the test results - * @param {Object} clientFactory a factory to spawn test clients - * @async - */ -async function startTest(number, message, clientArgs, updates, results, clientFactory) { - let count = 0; - for(let i in processes) { - i; // avoid eslint error - count++; - } - - const readyPromises = []; - if (count !== number) { - // launch clients - processes = {}; - for(let i = 0 ; i < number ; i++) { - launchClient(updates, results, clientFactory, readyPromises); - } - } - - // wait for all clients to have initialised - logger.info(`Waiting for ${readyPromises.length} clients to be ready... `); - - await Promise.all(readyPromises); - - logger.info(`${readyPromises.length} clients ready, starting test phase`); - - let txPerClient; - let totalTx = message.numb; - if (message.numb) { - // Run specified number of transactions - txPerClient = Math.floor(message.numb / number); - - // trim should be based on client number if specified with txNumber - if (message.trim) { - message.trim = Math.floor(message.trim / number); - } - - if(txPerClient < 1) { - txPerClient = 1; - } - message.numb = txPerClient; - } else if (message.txDuration) { - // Run for time specified txDuration based on clients - // Do nothing, we run for the time specified within message.txDuration - } else { - throw new Error('Unconditioned transaction rate driving mode'); - } - - message.clients = number; - - let promises = []; - let idx = 0; - for(let id in processes) { - let client = processes[id]; - let p = new Promise((resolve, reject) => { - client.obj.promise = { - resolve: resolve, - reject: reject - }; - }); - promises.push(p); - client.results = results; - client.updates = updates; - message.clientargs = clientArgs[idx]; - message.clientIdx = idx; - - if(totalTx % number !== 0 && idx === number-1){ - message.numb = totalTx - txPerClient*(number - 1); - } - - // send message to client and update idx - client.obj.send(message); - idx++; - } - - await Promise.all(promises); - // clear promises - for(let client in processes) { - delete client.promise; - } -} -module.exports.startTest = startTest; - -/** - * Send message to all child processes - * @param {JSON} message message - * @return {Number} number of child processes - */ -function sendMessage(message) { - for(let pid in processes) { - processes[pid].obj.send(message); - } - return processes.length; -} -module.exports.sendMessage = sendMessage; - -/** - * Stop all test clients(child processes) - */ -function stop() { - for(let pid in processes) { - processes[pid].obj.kill(); - } - processes = {}; -} -module.exports.stop = stop; diff --git a/packages/caliper-core/lib/client/zoo-util.js b/packages/caliper-core/lib/client/zoo-util.js deleted file mode 100644 index d7f73ad66..000000000 --- a/packages/caliper-core/lib/client/zoo-util.js +++ /dev/null @@ -1,265 +0,0 @@ -/* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -'use strict'; - -module.exports.NODE_ROOT = '/caliper'; -module.exports.NODE_CLIENT = '/caliper/clients'; - -const ZooKeeper = require('node-zookeeper-client'); -const logger = require('../utils/caliper-utils').getLogger('zoo-util.js'); -/** - * Check if specified znode exists - * @param {Object} zookeeper zk object - * @param {String} path path of znode - * @param {String} errLog specified informative error message - * @return {Promise} returned bool indicates whether the znode exists or not - */ -function exists(zookeeper, path, errLog) { - return new Promise((resolve, reject) => { - zookeeper.exists(path, (err, stat) => { - if(err) { - logger.error(errLog); - return reject(err); - } - if(stat) { - return resolve(true); - } - else { - return resolve(false); - } - }); - }); -} -module.exports.existsP = exists; - -/** - * Create a znode - * @param {Object} zookeeper zk object - * @param {String} path path of the znode - * @param {Buffer} data data of the znode - * @param {Number} mode create mode - * @param {String} errLog specified informative error message - * @return {Promise} path of the created znode - */ -function create(zookeeper, path, data, mode, errLog) { - return new Promise((resolve, reject) => { - zookeeper.create(path, data, ZooKeeper.ACL.OPEN_ACL_UNSAFE, mode, (err, path) => { - if(err) { - logger.error(errLog); - return reject(err); - } - else { - return resolve(path); - } - }); - }); -} -module.exports.createP = create; - -/** - * Remove a znode - * @param {Object} zookeeper zk object - * @param {String} path path of znode - * @param {Number} version znode's version - * @param {String} errLog specified informative error message - * @return {Promise} promise object - */ -function remove(zookeeper, path, version, errLog) { - return new Promise((resolve, reject)=>{ - zookeeper.remove(path, version, (err)=>{ - if(err) { - logger.error(errLog); - return reject(err); - } - return resolve(); - }); - }); -} -module.exports.removeP = remove; - -/** - * Read data of specified znode - * @param {Object} zookeeper zk object - * @param {String} path path of znode - * @param {Object} watcher zk watcher - * @param {String} errLog specified informative error message - * @return {Promise} data of specified znode - */ -function getData(zookeeper, path,watcher, errLog) { - return new Promise((resolve, reject) => { - zookeeper.getData(path, watcher, (err, data, stat) => { - if(err) { - logger.error(errLog); - return reject(err); - } - else { - return resolve(data); - } - }); - }); -} -module.exports.getDataP = getData; - -/** - * Get children under specified znode - * @param {Object} zookeeper zk object - * @param {String} path path of znode - * @param {Object} watcher zk watcher - * @param {String} errLog specified informative error message - * @return {Promise} children list - */ -function getChildren(zookeeper, path, watcher, errLog) { - return new Promise((resolve, reject) => { - zookeeper.getChildren(path, watcher, (err, children, stat)=>{ - if (err) { - logger.error(errLog); - return reject(err); - } - else { - return resolve(children); - } - }); - }); -} -module.exports.getChildrenP = getChildren; - -/** - * Remove all children under specified znode - * @param {Object} zookeeper zk object - * @param {String} path path of znode - * @param {String} errLog specified informative error message - * @return {Promise} promise object - */ -function removeChildren(zookeeper, path, errLog) { - return getChildren(zookeeper, path, null, '').then((children)=>{ - let promises = []; - children.forEach((child)=>{ - let p = remove(zookeeper, path+'/'+child, -1, ''); - promises.push(p); - }); - return Promise.all(promises); - }).catch((err)=>{ - logger.error(errLog); - logger.error(err); - return Promise.resolve(); - }); -} -module.exports.removeChildrenP = removeChildren; - -/** - * Receive and handle upcoming messages by watching specified znode continuously - * @param {Object} zookeeper zk object - * @param {String} path path of znode - * @param {Object} callback callback for new messages - * @param {String} errLog specified informative error message - * @return {Promise} promise object - */ -function watchMsgQueue(zookeeper, path, callback, errLog) { - let lastnode = null; - let cont = true; - /** - * main process - * @param {Object} resolve promise object - * @param {Object} reject promise object - */ - let whilst = ( resolve, reject) => { - getChildren( - zookeeper, - path, - (event)=>{ - if(cont) { - whilst(resolve, reject); - } - else { - logger.info('Stopped watching at '+path); - } - }, - errLog - ).then((children)=>{ - if(!cont) {return;} - if(children.length === 0) {return;} - children.sort(); // handle message one by one - let preKnown = lastnode; - lastnode = children[children.length - 1]; - let newidx = -1; - if(preKnown === null) { - newidx = 0; - } - else { // get recent unknown message - for(let i = 0 ; i < children.length ; i++) { - if(children[i] > preKnown) { - newidx = i; - break; - } - } - } - if(newidx < 0) { // no new message - return; - } - - let newNodes = children.slice(newidx); - newNodes.reduce( (prev, item) => { - return prev.then( () => { - return getData(zookeeper, path+'/'+item, null, 'Failed to getData from zookeeper'); - }).then((data)=>{ - zookeeper.remove(path+'/'+item, -1, (err)=>{ - if(err) { - logger.error(err.stack); - } - }); - - return callback(data); - }).then((stop)=>{ - if(stop) { - resolve(); - cont = false; - } - return Promise.resolve(); - }).catch((err)=>{ - return Promise.resolve(); - }); - }, Promise.resolve()); - }).catch((err)=>{ - logger.error(errLog); - cont = false; - resolve(); - }); - }; - - return new Promise((resolve, reject)=>{ - whilst( resolve, reject); - }); -} -module.exports.watchMsgQueueP = watchMsgQueue; - -/** - * Generate in node - * @param {String} clientID identity of specified zk client - * @return {String} path of the in node - */ -function getInNode(clientID) { - return this.NODE_ROOT+'/'+clientID+'_in'; -} -module.exports.getInNode = getInNode; - -/** - * Generate out node - * @param {String} clientID identity of specified zk client - * @return {String} path of the out node - */ -function getOutNode(clientID) { - return this.NODE_ROOT+'/'+clientID+'_out'; -} -module.exports.getOutNode = getOutNode; diff --git a/packages/caliper-core/lib/gui/README.md b/packages/caliper-core/lib/gui/README.md deleted file mode 100644 index 3d3616415..000000000 --- a/packages/caliper-core/lib/gui/README.md +++ /dev/null @@ -1,20 +0,0 @@ -This is an implementation of temporary GUI demo. - -SSH is used to start local/remote benchmark and fetch results (which are updated in temporary log files) periodically, libssh2-php is required to set up ssh connection. -Go to www/remotecontrol.php to set the host address and login name/password. - -EventSource is used in this demo. IE may not support EventSource, so we recommend using Chrome to run the demo. - -A simple script `caliper/scripts/start.sh` is used to start a benchmark, you may need to add dependent environment such as GOPATH in it. - -Only 'simple' benchmark is integrated into the GUI. The supported benchmark is hard coded now. - -The demo may support fetch available benchmarks, as well as corresponding configuration files dynamically in the future version. - -The demo may also support upload user defined configuration files for specified benchmark in the future. - -Directory structure: -* ./www, contains the web portal and php files. -* ./src, contains scripts for caliper benchmark to update performance related log files -* ./output, contains log files mentioned before -* caliper/scripts/start.sh, starts local benchmark \ No newline at end of file diff --git a/packages/caliper-core/lib/gui/src/demo.js b/packages/caliper-core/lib/gui/src/demo.js deleted file mode 100644 index 847de85c0..000000000 --- a/packages/caliper-core/lib/gui/src/demo.js +++ /dev/null @@ -1,231 +0,0 @@ -/* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - - -'use strict'; - -const logger = require('../../utils/caliper-utils').getLogger('demo.js'); -const fs = require('fs-extra'); -const {join} = require('path'); -const demoPath = '/tmp/caliper/output'; -const demoFile = join(demoPath, 'demo.json'); - -let demoInterval = 1; // interval length(s) -let demoXLen = 60; // default x axis length -let demoData; -let demoInterObj = null; - -/** - * Demonstration - */ -function demoInit() { - demoData = { - maxlen : 300, - throughput: { - x: [], - submitted: [0], - succeeded: [0], - failed: [0] - }, - latency: { - x: [], - max: [0], - min: [0], - avg: [0] - }, - summary: { - txSub: 0, - txSucc: 0, - txFail: 0, - round: 0, - }, - report: '' - }; - - for(let i = 0 ; i < demoXLen ; i++) { - demoData.throughput.x.push(i * demoInterval); - demoData.latency.x.push(i * demoInterval); - } - - if (!fs.existsSync(demoPath)) { - fs.mkdirpSync(demoPath); - } - - fs.writeFileSync(demoFile, JSON.stringify(demoData)); -} -module.exports.init = demoInit; -/** - * Add Throughput - * @param {*} sub submitted - * @param {*} suc successful - * @param {*} fail fail - */ -function demoAddThroughput(sub, suc, fail) { - demoData.throughput.submitted.push(sub/demoInterval); - demoData.throughput.succeeded.push(suc/demoInterval); - demoData.throughput.failed.push(fail/demoInterval); - if (demoData.throughput.x.length < demoData.throughput.submitted.length) { - let last = demoData.throughput.x[demoData.throughput.x.length - 1]; - demoData.throughput.x.push(last + demoInterval); - } - if (demoData.throughput.submitted.length > demoData.maxlen) { - demoData.throughput.submitted.shift(); - demoData.throughput.succeeded.shift(); - demoData.throughput.failed.shift(); - demoData.throughput.x.shift(); - } - demoData.summary.txSub += sub; - demoData.summary.txSucc += suc; - demoData.summary.txFail += fail; -} - -/** - * Add Latency - * @param {*} max the maximum - * @param {*} min the minimum - * @param {*} avg the average - */ -function demoAddLatency(max, min, avg) { - demoData.latency.max.push(max); - demoData.latency.min.push(min); - demoData.latency.avg.push(avg); - if(demoData.latency.x.length < demoData.latency.max.length) { - let last = demoData.latency.x[demoData.latency.x.length - 1]; - demoData.latency.x.push(last + demoInterval); - } - if (demoData.latency.max.length > demoData.maxlen) { - demoData.latency.max.shift(); - demoData.latency.min.shift(); - demoData.latency.avg.shift(); - demoData.latency.x.shift(); - } -} - -/** - * Refresh addat - * @param {*} updates updates to use - */ -function demoRefreshData(updates) { - if(updates.length === 0) { - demoAddThroughput(0,0,0); - demoAddLatency(0,0,0); - } - else { - let sub = 0, suc = 0, fail = 0; - let deMax = -1, deMin = -1, deAvg = 0; - for(let i = 0 ; i < updates.length ; i++) { - let data = updates[i]; - sub += data.submitted; - suc += data.committed.succ; - fail += data.committed.fail; - - if(data.committed.succ > 0) { - if(deMax === -1 || deMax < data.committed.delay.max) { - deMax = data.committed.delay.max; - } - if(deMin === -1 || deMin > data.committed.delay.min) { - deMin = data.committed.delay.min; - } - deAvg += data.committed.delay.sum; - } - } - if(suc > 0) { - deAvg /= suc; - } - demoAddThroughput(sub, suc, fail); - - if(isNaN(deMax) || isNaN(deMin) || deAvg === 0) { - demoAddLatency(0,0,0); - } - else { - demoAddLatency(deMax, deMin, deAvg); - } - - } - - logger.info('[Transaction Info] - Submitted: ' + demoData.summary.txSub + - ' Succ: ' + demoData.summary.txSucc + - ' Fail:' + demoData.summary.txFail + - ' Unfinished:' + (demoData.summary.txSub - demoData.summary.txSucc - demoData.summary.txFail)); - - let fs = require('fs'); - fs.writeFileSync(demoFile, JSON.stringify(demoData)); -} - -let client; -let updateTail = 0; -let updateID = 0; - -/** - * Perform an update - */ -function update() { - if (typeof client === 'undefined') { - demoRefreshData([]); - return; - } - let updates = client.getUpdates(); - if(updates.id > updateID) { // new buffer - updateTail = 0; - updateID = updates.id; - } - let data = []; - let len = updates.data.length; - if(len > updateTail) { - data = updates.data.slice(updateTail, len); - updateTail = len; - } - demoRefreshData(data); -} - -/** - * demoStartWatch - * @param {*} clientObj the client object - */ -function demoStartWatch(clientObj) { - //demoProcesses = processes.slice(); - client = clientObj; - if(demoInterObj === null) { - updateTail = 0; - updateID = 0; - // start a interval to query updates - demoInterObj = setInterval(update, demoInterval * 1000); - } -} -module.exports.startWatch = demoStartWatch; - -/** - * demoPauseWatch - */ -function demoPauseWatch() { - demoData.summary.round += 1; -} - -module.exports.pauseWatch = demoPauseWatch; - -/** - * demoStopWatch - * @param {*} output the output - */ -function demoStopWatch(output) { - if(demoInterObj) { - clearInterval(demoInterObj); - demoInterObj = null; - } - demoData.report = output; - update(); -} - -module.exports.stopWatch = demoStopWatch; - diff --git a/packages/caliper-core/lib/gui/www/remotecontrol.php b/packages/caliper-core/lib/gui/www/remotecontrol.php deleted file mode 100644 index 87d1afaa0..000000000 --- a/packages/caliper-core/lib/gui/www/remotecontrol.php +++ /dev/null @@ -1,103 +0,0 @@ -$name, 'data'=>$data); - echo "data:" . json_encode($data, JSON_UNESCAPED_SLASHES) . "\n\n"; - @ob_flush(); - @flush(); - } - - if(ssh2_auth_password($connect, $user, $pwd)){ - if($debug) { - sendmsg('debug', 'ssh connected, try to run: bash '.$path.'scripts/start.sh ' . $_GET['b'] . ' ' . $_GET['s']); - } - - // start the benchmark - $stream = ssh2_exec($connect, 'bash '.$path.'scripts/start.sh ' . $_GET['b'] . ' ' . $_GET['s']); - stream_set_blocking($stream, true); - // fetch the log file to get running result - while($stream) { - @session_start(); - if($_SESSION['started'] == false) { - sendmsg("finish", "stopped"); - exit(); - } - session_write_close(); - - sleep(1); - $out = ssh2_exec($connect, 'cat '.$path.'output.log'); - stream_set_blocking($out, true); - $result = stream_get_contents($out); - sendmsg("log", $result); - fclose($out); - - $demo = ssh2_exec($connect, 'cat '.$path.'src/gui/output/demo.json'); - stream_set_blocking($demo, true); - $demoResult = stream_get_contents($demo); - sendmsg("metrics", $demoResult); - if(!$hasReport) { - $json = json_decode($demoResult); - $report = $json->report; - if(strpos($report, '.html') !== false ) { - $html = ssh2_exec($connect, 'cat '.$report); - stream_set_blocking($html, true); - $htmlData = stream_get_contents($html); - $htmlFile = fopen("report.html", "w"); - fwrite($htmlFile, $htmlData); - fclose($htmlFile); - fclose($html); - $hasReport = true; - sendmsg("report", "ok"); - } - } - fclose($demo); - - if(strpos($result, '# fail ') !== false || strpos($result, '# ok') !== false) { - sleep(2); - break; - } - } - - stream_get_contents($stream); - fclose($stream); - - sendmsg('finish','ok'); - exit(); - } - else { - sendmsg('finish', 'Error: could not connect to ssh server'); - exit(); - } - } - catch(Exception $e) { - sendmsg('finish', 'Error: '.$e->getMessage()); - exit(); - } - -?> \ No newline at end of file diff --git a/packages/caliper-core/lib/gui/www/stop.php b/packages/caliper-core/lib/gui/www/stop.php deleted file mode 100644 index 6f021c93b..000000000 --- a/packages/caliper-core/lib/gui/www/stop.php +++ /dev/null @@ -1,5 +0,0 @@ - diff --git a/packages/caliper-core/lib/monitor/monitor-docker.js b/packages/caliper-core/lib/monitor/monitor-docker.js index 65b5d263e..817310163 100644 --- a/packages/caliper-core/lib/monitor/monitor-docker.js +++ b/packages/caliper-core/lib/monitor/monitor-docker.js @@ -16,125 +16,13 @@ 'use strict'; const Util = require('../utils/caliper-utils'); -const logger = Util.getLogger('monitor-docker.js'); +const Logger = Util.getLogger('monitor-docker'); const MonitorInterface = require('./monitor-interface'); +const MonitorUtilities = require('./monitor-utilities'); -/** - * create a containerStat object - * @return {JSON} containerStat object - */ -function newContainerStat() { - return { - mem_usage: [], - mem_percent: [], - cpu_percent: [], - netIO_rx: [], - netIO_tx: [], - blockIO_rx: [], - blockIO_wx: [] - }; -} - -/** - * Find local containers according to searching filters - * @return {Promise} promise object - */ -function findContainers() { - let filterName = {local:[], remote:{}}; - let url = require('url'); - if(this.filter.hasOwnProperty('name')) { - for(let key in this.filter.name) { - let name = this.filter.name[key]; - if(name.indexOf('http://') === 0) { - let remote = url.parse(name, true); - if(remote.hostname === null || remote.port === null || remote.pathname === '/') { - logger.warn('monitor-docker: unrecognized host, ' + name); - } - else if(filterName.remote.hasOwnProperty(remote.hostname)) { - filterName.remote[remote.hostname].containers.push(remote.pathname); - } - else { - filterName.remote[remote.hostname] = {port: remote.port, containers: [remote.pathname]}; - } - } - else{ - filterName.local.push(name); - } - } - } - - let promises = []; - - // find local containers by name - if(filterName.local.length > 0) { - let p = this.si.dockerContainers('active').then((containers) => { - let size = containers.length; - if(size === 0) { - logger.error('monitor-docker: could not find active local container'); - return Promise.resolve(); - } - if(filterName.local.indexOf('all') !== -1) { - for(let i = 0 ; i < size ; i++){ - this.containers.push({id: containers[i].id, name: containers[i].name, remote: null}); - this.stats[containers[i].id] = newContainerStat(); - } - } - else { - for(let i = 0 ; i < size ; i++){ - if(filterName.local.indexOf(containers[i].name) !== -1) { - this.containers.push({id: containers[i].id, name: containers[i].name, remote: null}); - this.stats[containers[i].id] = newContainerStat(); - } - } - } - - return Promise.resolve(); - }).catch((err) => { - logger.error('Error(monitor-docker):' + err); - return Promise.resolve(); - }); - promises.push(p); - } - // find remote containers by name - for(let h in filterName.remote) { - let docker = new this.Docker({ - host: h, - port: filterName.remote[h].port - // version: 'v1.20' - }); - let p = docker.listContainers().then((containers) => { - let size = containers.length; - if(size === 0) { - logger.error('monitor-docker: could not find remote container at ' + h); - return Promise.resolve(); - } - - if(filterName.remote[h].containers.indexOf('/all') !== -1) { - for(let i = 0 ; i < size ; i++) { - let container = docker.getContainer(containers[i].Id); - this.containers.push({id: containers[i].Id, name: h + containers[i].Names[0], remote: container}); - this.stats[containers[i].Id] = newContainerStat(); - } - } - else { - for(let i = 0 ; i < size ; i++) { - if(filterName.remote[h].containers.indexOf(containers[i].Names[0]) !== -1) { - let container = docker.getContainer(containers[i].Id); - this.containers.push({id: containers[i].Id, name: h + containers[i].Names[0], remote: container}); - this.stats[containers[i].Id] = newContainerStat(); - } - } - } - return Promise.resolve(); - }).catch((err) => { - logger.error('Error(monitor-docker):' + err); - return Promise.resolve(); - }); - promises.push(p); - } - - return Promise.all(promises); -} +const URL = require('url'); +const Docker = require('dockerode'); +const SystemInformation = require('systeminformation'); /** * Resource monitor for local/remote docker containers @@ -142,19 +30,16 @@ function findContainers() { class MonitorDocker extends MonitorInterface { /** * Constructor - * @param {JSON} filter lookup filter for containers + * @param {JSON} monitorConfig Configuration object for the monitor * @param {*} interval resource fetching interval */ - constructor(filter, interval) { - super(filter, interval); - this.si = require('systeminformation'); - this.Docker = require('dockerode'); - this.containers = []; // {id, name, obj} + constructor(monitorConfig, interval) { + super(monitorConfig, interval); + this.containers = null; this.isReading = false; this.intervalObj = null; this.stats = {'time': []}; - this.hasContainters = findContainers.call(this); - /* this.stats : record statistics of each container + /* this.stats : used to record statistics of each container { 'time' : [] // time slot 'container_id" : { // refer to https://www.npmjs.com/package/systeminformation @@ -173,180 +58,325 @@ class MonitorDocker extends MonitorInterface { } /** - * Start the monitor - * @return {Promise} promise object + * Find local containers according to searching filters and persist in local memory + * @async */ - start() { - return this.hasContainters.then( () => { - let self = this; - /** - * callback for reading containers' resouce usage - */ - function readContainerStats() { - if(self.isReading) { - return; - } - self.isReading = true; - let statPromises = []; - for(let i = 0 ;i < self.containers.length ; i++){ - if(self.containers[i].remote === null) { // local - statPromises.push(self.si.dockerContainerStats(self.containers[i].id)); + async findContainers() { + this.containers = []; + let filterName = {local:[], remote:{}}; + // Split docker items that are local or remote + if(this.monitorConfig.hasOwnProperty('name')) { + for(let key in this.monitorConfig.name) { + let name = this.monitorConfig.name[key]; + if(name.indexOf('http://') === 0) { + // Is remote + let remote = URL.parse(name, true); + if(remote.hostname === null || remote.port === null || remote.pathname === '/') { + Logger.warn('unrecognized host, ' + name); + } else if(filterName.remote.hasOwnProperty(remote.hostname)) { + filterName.remote[remote.hostname].containers.push(remote.pathname); + } else { + filterName.remote[remote.hostname] = {port: remote.port, containers: [remote.pathname]}; } - else { // remote - statPromises.push(self.containers[i].remote.stats({stream: false})); + } else { + // Is local + filterName.local.push(name); + } + } + } + + // Filter local containers by name + if(filterName.local.length > 0) { + try { + const containers = await SystemInformation.dockerContainers('active'); + let size = containers.length; + if(size === 0) { + Logger.error('Could not find any active local containers'); + } else { + if(filterName.local.indexOf('all') !== -1) { + // Add all containers + for(let i = 0 ; i < size ; i++){ + this.containers.push({id: containers[i].id, name: containers[i].name, remote: null}); + this.stats[containers[i].id] = this.newContainerStat(); + } + } else { + // Filter containers + for(let i = 0 ; i < size ; i++){ + if(filterName.local.indexOf(containers[i].name) !== -1) { + this.containers.push({id: containers[i].id, name: containers[i].name, remote: null}); + this.stats[containers[i].id] = this.newContainerStat(); + } + } } } - Promise.all(statPromises).then((results) => { - self.stats.time.push(Date.now()/1000); - for(let i = 0 ; i < results.length ; i++) { - let stat = results[i]; - let id = stat.id; - if(self.containers.length <= i) { - break; + } catch (error) { + Logger.error(`Error retrieving local containers: ${error}`); + } + } + // Filter remote containers by name + for(let h in filterName.remote) { + try { + // Instantiate for the host/port + let docker = new Docker({ + host: h, + port: filterName.remote[h].port + }); + + // Retrieve and filter containers + const containers = await docker.listContainers(); + if (containers.length === 0) { + Logger.error('monitor-docker: could not find remote container at ' + h); + } else { + if(filterName.remote[h].containers.indexOf('/all') !== -1) { + for(let i = 0 ; i < containers.length ; i++) { + let container = docker.getContainer(containers[i].Id); + this.containers.push({id: containers[i].Id, name: h + containers[i].Names[0], remote: container}); + this.stats[containers[i].Id] = this.newContainerStat(); } - if(id !== self.containers[i].id) { - logger.warn('monitor-docker: inconsistent id'); - continue; + } else { + for(let i = 0 ; i < containers.length ; i++) { + if(filterName.remote[h].containers.indexOf(containers[i].Names[0]) !== -1) { + let container = docker.getContainer(containers[i].Id); + this.containers.push({id: containers[i].Id, name: h + containers[i].Names[0], remote: container}); + this.stats[containers[i].Id] = this.newContainerStat(); + } } - if(self.containers[i].remote === null) { // local - self.stats[id].mem_usage.push(stat.mem_usage); - self.stats[id].mem_percent.push(stat.mem_percent); - let cpuDelta = stat.cpu_stats.cpu_usage.total_usage - stat.precpu_stats.cpu_usage.total_usage; - let sysDelta = stat.cpu_stats.system_cpu_usage - stat.precpu_stats.system_cpu_usage; - if(cpuDelta > 0 && sysDelta > 0) { - if(stat.cpu_stats.cpu_usage.hasOwnProperty('percpu_usage') && stat.cpu_stats.cpu_usage.percpu_usage !== null) { - self.stats[id].cpu_percent.push(cpuDelta / sysDelta * MonitorDocker.coresInUse(stat.cpu_stats) * 100.0); - } - else { - self.stats[id].cpu_percent.push(cpuDelta / sysDelta * 100.0); - } + } + } + } catch (error) { + Logger.error(`Error retrieving remote containers: ${error}`); + } + } + } + + /** + * Create and return a containerStat object + * @return {JSON} containerStat object + */ + newContainerStat() { + return { + mem_usage: [], + mem_percent: [], + cpu_percent: [], + netIO_rx: [], + netIO_tx: [], + blockIO_rx: [], + blockIO_wx: [] + }; + } + + /** + * Callback for reading containers' resource usage + * @async + */ + async readContainerStats() { + // Prevent overlapping read of stats + if (this.isReading) { + return; + } else { + this.isReading = true; + let startPromises = []; + try { + for (let i = 0 ;i < this.containers.length ; i++){ + if (this.containers[i].remote === null) { + // local + startPromises.push(SystemInformation.dockerContainerStats(this.containers[i].id)); + } else { + // remote + startPromises.push(this.containers[i].remote.stats({stream: false})); + } + } + + const results = await Promise.all(startPromises); + this.stats.time.push(Date.now()/1000); + for (let i = 0 ; i < results.length ; i++) { + let stat = results[i]; + let id = stat.id; + if (this.containers.length <= i) { + break; + } + if (id !== this.containers[i].id) { + Logger.warn('inconsistent id within statistics gathering'); + continue; + } + if (this.containers[i].remote === null) { + // local + this.stats[id].mem_usage.push(stat.mem_usage); + this.stats[id].mem_percent.push(stat.mem_percent); + let cpuDelta = stat.cpu_stats.cpu_usage.total_usage - stat.precpu_stats.cpu_usage.total_usage; + let sysDelta = stat.cpu_stats.system_cpu_usage - stat.precpu_stats.system_cpu_usage; + if (cpuDelta > 0 && sysDelta > 0) { + if(stat.cpu_stats.cpu_usage.hasOwnProperty('percpu_usage') && stat.cpu_stats.cpu_usage.percpu_usage !== null) { + this.stats[id].cpu_percent.push(cpuDelta / sysDelta * this.coresInUse(stat.cpu_stats) * 100.0); + } else { + this.stats[id].cpu_percent.push(cpuDelta / sysDelta * 100.0); } - else { - self.stats[id].cpu_percent.push(0); + } else { + this.stats[id].cpu_percent.push(0); + } + this.stats[id].netIO_rx.push(stat.netIO.rx); + this.stats[id].netIO_tx.push(stat.netIO.tx); + this.stats[id].blockIO_rx.push(stat.blockIO.r); + this.stats[id].blockIO_wx.push(stat.blockIO.w); + } else { + // remote + this.stats[id].mem_usage.push(stat.memory_stats.usage); + this.stats[id].mem_percent.push(stat.memory_stats.usage / stat.memory_stats.limit); + //this.stats[id].cpu_percent.push((stat.cpu_stats.cpu_usage.total_usage - stat.precpu_stats.cpu_usage.total_usage) / (stat.cpu_stats.system_cpu_usage - stat.precpu_stats.system_cpu_usage) * 100); + let cpuDelta = stat.cpu_stats.cpu_usage.total_usage - stat.precpu_stats.cpu_usage.total_usage; + let sysDelta = stat.cpu_stats.system_cpu_usage - stat.precpu_stats.system_cpu_usage; + if (cpuDelta > 0 && sysDelta > 0) { + if (stat.cpu_stats.cpu_usage.hasOwnProperty('percpu_usage') && stat.cpu_stats.cpu_usage.percpu_usage !== null) { + // this.stats[id].cpu_percent.push(cpuDelta / sysDelta * stat.cpu_stats.cpu_usage.percpu_usage.length * 100.0); + this.stats[id].cpu_percent.push(cpuDelta / sysDelta * this.coresInUse(stat.cpu_stats) * 100.0); + } else { + this.stats[id].cpu_percent.push(cpuDelta / sysDelta * 100.0); } - self.stats[id].netIO_rx.push(stat.netIO.rx); - self.stats[id].netIO_tx.push(stat.netIO.tx); - self.stats[id].blockIO_rx.push(stat.blockIO.r); - self.stats[id].blockIO_wx.push(stat.blockIO.w); + } else { + this.stats[id].cpu_percent.push(0); } - else { // remote - self.stats[id].mem_usage.push(stat.memory_stats.usage); - self.stats[id].mem_percent.push(stat.memory_stats.usage / stat.memory_stats.limit); - //self.stats[id].cpu_percent.push((stat.cpu_stats.cpu_usage.total_usage - stat.precpu_stats.cpu_usage.total_usage) / (stat.cpu_stats.system_cpu_usage - stat.precpu_stats.system_cpu_usage) * 100); - let cpuDelta = stat.cpu_stats.cpu_usage.total_usage - stat.precpu_stats.cpu_usage.total_usage; - let sysDelta = stat.cpu_stats.system_cpu_usage - stat.precpu_stats.system_cpu_usage; - if(cpuDelta > 0 && sysDelta > 0) { - if(stat.cpu_stats.cpu_usage.hasOwnProperty('percpu_usage') && stat.cpu_stats.cpu_usage.percpu_usage !== null) { - // self.stats[id].cpu_percent.push(cpuDelta / sysDelta * stat.cpu_stats.cpu_usage.percpu_usage.length * 100.0); - self.stats[id].cpu_percent.push(cpuDelta / sysDelta * MonitorDocker.coresInUse(stat.cpu_stats) * 100.0); - } - else { - self.stats[id].cpu_percent.push(cpuDelta / sysDelta * 100.0); + let ioRx = 0, ioTx = 0; + for (let eth in stat.networks) { + ioRx += stat.networks[eth].rx_bytes; + ioTx += stat.networks[eth].tx_bytes; + } + this.stats[id].netIO_rx.push(ioRx); + this.stats[id].netIO_tx.push(ioTx); + let diskR = 0, diskW = 0; + if (stat.blkio_stats && stat.blkio_stats.hasOwnProperty('io_service_bytes_recursive')){ + //Logger.debug(stat.blkio_stats.io_service_bytes_recursive); + let temp = stat.blkio_stats.io_service_bytes_recursive; + for (let dIo =0; dIo { - logger.error(err); - self.isReading = false; - }); + } + this.isReading = false; + } catch (error) { + Logger.error(`Error reading monitor statistics: ${error}`); + this.isReading = false; } + } + } - readContainerStats(); // read stats immediately - this.intervalObj = setInterval(readContainerStats, this.interval); - return Promise.resolve(); - }).catch((err) => { - return Promise.reject(err); - }); + /** + * Start the monitor + * @async + */ + async start() { + // Conditionally build monitored containers, these are persisted between rounds and restart action + if (!this.containers) { + await this.findContainers(); + } + // Read stats immediately, then kick off monitor refresh at interval + await this.readContainerStats(); + let self = this; + this.intervalObj = setInterval(async () => { await self.readContainerStats(); }, this.interval); } /** * Restart the monitor - * @return {Promise} promise object + * @async */ - restart() { + async restart() { clearInterval(this.intervalObj); - for(let key in this.stats) { - if(key === 'time') { + for (let key in this.stats) { + if (key === 'time') { this.stats[key] = []; - } - else { + } else { for(let v in this.stats[key]) { this.stats[key][v] = []; } } } - return this.start(); + await this.start(); } /** * Stop the monitor - * @return {Promise} promise object + * @async */ - stop() { + async stop() { clearInterval(this.intervalObj); this.containers = []; this.stats = {'time': []}; + await Util.sleep(100); + } + + /** + * Get a Map of result items + * @return {Map} Map of items to build results for, with default null entries + */ + getResultColumnMap() { + const columns = ['Type', 'Name', 'Memory(max)', 'Memory(avg)', 'CPU% (max)', 'CPU% (avg)', 'Traffic In', 'Traffic Out', 'Disc Read','Disc Write']; + const resultMap = new Map(); - return Util.sleep(100); + for (const item of columns) { + resultMap.set(item, 'N/A'); + } + + // This is a Docker Type, so set here + resultMap.set('Type', 'Docker'); + return resultMap; } /** - * Get information of watched containers - * info = { - * key: key of the container - * info: { - * TYPE: 'docker', - * NAME: name of the container - * } - * } - * @return {Array} array of containers' information + * Get statistics from the monitor in the form of an Array containing Map detailing key/value pairs + * @return {Map[]} an array of resource maps for watched containers + * @async */ - getPeers() { - let info = []; - for(let i in this.containers) { - let c = this.containers[i]; - if(c.hasOwnProperty('id')) { - info.push({ - 'key' : c.id, - 'info' : { - 'TYPE' : 'Docker', - 'NAME' : c.name - } - }); + async getStatistics() { + try { + const watchItemStats = []; + // Build a statistic for each monitored container and push into watchItems array + for (const container of this.containers) { + if (container.hasOwnProperty('id')) { + + // Grab the key + const key = container.id; + + // retrieve stats for the key + let mem = this.getMemHistory(key); + let cpu = this.getCpuHistory(key); + let net = this.getNetworkHistory(key); + let disc = this.getDiscHistory(key); + let mem_stat = MonitorUtilities.getStatistics(mem); + let cpu_stat = MonitorUtilities.getStatistics(cpu); + + // Store in a Map + const watchItemStat = this.getResultColumnMap(); + watchItemStat.set('Name', container.name); + watchItemStat.set('Memory(max)', MonitorUtilities.byteNormalize(mem_stat.max)); + watchItemStat.set('Memory(avg)', MonitorUtilities.byteNormalize(mem_stat.avg)); + watchItemStat.set('CPU% (max)', cpu_stat.max.toFixed(2)); + watchItemStat.set('CPU% (avg)', cpu_stat.avg.toFixed(2)); + watchItemStat.set('Traffic In', MonitorUtilities.byteNormalize((net.in[net.in.length-1] - net.in[0]))); + watchItemStat.set('Traffic Out', MonitorUtilities.byteNormalize((net.out[net.out.length-1] - net.out[0]))); + watchItemStat.set('Disc Write', MonitorUtilities.byteNormalize((disc.write[disc.write.length-1] - disc.write[0]))); + watchItemStat.set('Disc Read', MonitorUtilities.byteNormalize((disc.read[disc.read.length-1] - disc.read[0]))); + + // append return array + watchItemStats.push(watchItemStat); + } } + return watchItemStats; + } catch(error) { + Logger.error('Failed to read monitoring data, ' + (error.stack ? error.stack : error)); + return []; } - return info; } + /** * Get history of memory usage * @param {String} key key of the container @@ -371,16 +401,16 @@ class MonitorDocker extends MonitorInterface { * @return {Array} array of network IO usage */ getNetworkHistory(key) { - return {'in': this.stats[key].netIO_rx, 'out':this.stats[key].netIO_tx}; + return {'in': this.stats[key].netIO_rx, 'out': this.stats[key].netIO_tx}; } /** - * Get history of disc usage as {read, wrtie} + * Get history of disc usage as {read, write} * @param {String} key key of the container * @return {Array} array of disc usage */ getDiscHistory(key) { - return {'read': this.stats[key].blockIO_rx, 'write':this.stats[key].blockIO_wx}; + return {'read': this.stats[key].blockIO_rx, 'write': this.stats[key].blockIO_wx}; } /** @@ -388,8 +418,8 @@ class MonitorDocker extends MonitorInterface { * @param {json} cpu_stats the statistics of cpu * @return {number} the number core in real use */ - static coresInUse(cpu_stats) { - return cpu_stats.online_cpus || MonitorDocker.findCoresInUse(cpu_stats.cpu_usage.percpu_usage || []); + coresInUse(cpu_stats) { + return cpu_stats.online_cpus || this.findCoresInUse(cpu_stats.cpu_usage.percpu_usage || []); } /** @@ -397,7 +427,7 @@ class MonitorDocker extends MonitorInterface { * @param {array} percpu_usage the usage cpu array * @return {number} the the percpu_usage.length */ - static findCoresInUse(percpu_usage) { + findCoresInUse(percpu_usage) { percpu_usage = percpu_usage.filter((coreUsage) => { if (coreUsage > 0) { return (coreUsage); @@ -405,7 +435,5 @@ class MonitorDocker extends MonitorInterface { }); return percpu_usage.length; } - - } module.exports = MonitorDocker; diff --git a/packages/caliper-core/lib/monitor/monitor-interface.js b/packages/caliper-core/lib/monitor/monitor-interface.js index 9a0debd70..bf6e0cd64 100644 --- a/packages/caliper-core/lib/monitor/monitor-interface.js +++ b/packages/caliper-core/lib/monitor/monitor-interface.js @@ -22,72 +22,41 @@ class MonitorInterface{ /** * Constructor - * @param {JSON} filter Lookup filter - * @param {*} interval Watching interval, in second + * @param {JSON} monitorConfig Configuration object for the monitor + * @param {number} interval Watching interval, in seconds */ - constructor(filter, interval) { - this.filter = filter; - this.interval = interval*1000; // ms + constructor(monitorConfig, interval) { + this.monitorConfig = monitorConfig; + this.interval = interval*1000; // convert to ms } /** * start monitoring */ - start() { + async start() { throw new Error('start is not implemented for this monitor'); } /** * restart monitoring */ - restart() { + async restart() { throw new Error('restart is not implemented for this monitor'); } /** * stop monitoring */ - stop() { + async stop() { throw new Error('stop is not implemented for this monitor'); } /** - * Get watching list - */ - getPeers() { - throw new Error('getPeers is not implemented for this monitor'); - } - - /** - * Get history of memory usage, in byte - * @param {String} key Lookup key - */ - getMemHistory(key) { - throw new Error('getMemHistory is not implemented for this monitor'); - } - - /** - * Get history of cpu usage, % - * @param {String} key Lookup key - */ - getCpuHistory(key) { - throw new Error('getCpuHistory is not implemented for this monitor'); - } - - /** - * Get history of network IO usage, byte - * @param {String} key Lookup key - */ - getNetworkHistory(key) { - throw new Error('getNetworkHistory is not implemented for this monitor'); - } - - /** - * Get history of disc usage as {read, wrtie} - * @param {String} key Lookup key + * Get statistics from the monitor in the form of an Array containing Map detailing key/value pairs + * @async */ - getDiscHistory(key) { - throw new Error('getDiscHistory is not implemented for this monitor'); + async getStatistics() { + throw new Error('getStatistics is not implemented for this monitor'); } } module.exports = MonitorInterface; diff --git a/packages/caliper-core/lib/monitor/monitor-orchestrator.js b/packages/caliper-core/lib/monitor/monitor-orchestrator.js new file mode 100644 index 000000000..19081352f --- /dev/null +++ b/packages/caliper-core/lib/monitor/monitor-orchestrator.js @@ -0,0 +1,158 @@ +/* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + + +'use strict'; + +const DockerMonitor = require('./monitor-docker.js'); +const ProcessMonitor = require('./monitor-process.js'); +const PrometheusMonitor = require('./monitor-prometheus.js'); +const Util = require('../utils/caliper-utils'); +const logger= Util.getLogger('monitor.js'); + +const DOCKER = 'docker'; +const PROCESS = 'process'; +const PROMETHEUS = 'prometheus'; +const VALID_MONITORS = [DOCKER, PROCESS, PROMETHEUS]; + +/** + * MonitorOrchestrator class, containing a map of user specified monitor types and operations to interact with the Monitor interface that they implement + */ +class MonitorOrchestrator { + /** + * Constructor + * @param {String} configPath path of the configuration file + */ + constructor(configPath) { + this.config = Util.parseYaml(configPath); + this.started = false; + this.monitors = new Map(); + + // Parse the config and retrieve the monitor types + const m = this.config.monitor; + if(typeof m === 'undefined') { + throw new Error('Failed to find a monitor in the config file'); + } + + if(typeof m.type === 'undefined') { + throw new Error('Failed to find monitor types in config file'); + } + + const monitorTypes = Array.isArray(m.type) ? m.type : [m.type]; + for(let type of monitorTypes) { + let monitor = null; + if(type === DOCKER) { + monitor = new DockerMonitor(m.docker, m.interval); + } else if(type === PROCESS) { + monitor = new ProcessMonitor(m.process, m.interval); + } else if(type === PROMETHEUS) { + monitor = new PrometheusMonitor(m.prometheus); + } else { + const msg = `Unsupported monitor type ${type}, must be one of ${VALID_MONITORS}`; + logger.error(msg); + throw new Error(msg); + } + this.monitors.set(type, monitor); + } + } + + + /** + * Get all monitor types stored by the orchestrator + * @returns {IterableIterator} iterator of all monitor types + */ + getAllMonitorTypes(){ + return this.monitors.keys(); + } + + /** + * Check if a specific monitor exists + * @param {String} type the type name of the monitor to retrieve + * @returns {Boolean} true if a monitor of the named type exists + */ + hasMonitor(type){ + return this.monitors.has(type); + } + + /** + * Retrieve a monitor with the passed name + * @param {String} type the type name of the monitor to retrieve + * @returns {Monitor} a monitor of the named type + */ + getMonitor(type){ + if(this.hasMonitor(type)){ + return this.monitors.get(type); + } else { + throw new Error(`No monitor of type ${type} available for retrieval from orchestrator`); + } + } + + /** + * Start the monitors held by the orchestrator + * @async + */ + async startAllMonitors() { + + if(this.started === false) { + + for (let key of this.monitors.keys()) { + const monitor = this.monitors.get(key); + await monitor.start(); + } + + this.started = true; + } else { + await this.restartAllMonitors(); + } + } + + /** + * stop the all the stored monitors + * @async + */ + async stopAllMonitors() { + if(this.started === true) { + + for (let key of this.monitors.keys()) { + const monitor = this.monitors.get(key); + await monitor.stop(); + } + + this.started = false; + } + } + + /** + * Restart all monitors, all data recorded internally will be cleared + * @async + */ + async restartAllMonitors() { + for (let key of this.monitors.keys()) { + const monitor = this.monitors.get(key); + await monitor.restart(); + } + } + + /** + * Get an Array of statistics maps for a named resource monitor + * @param {String} type the monitor type + * @return {Map[]} an array of resource maps + * @async + */ + async getStatisticsForMonitor(type) { + return await this.monitors.get(type).getStatistics(); + } +} + +module.exports = MonitorOrchestrator; diff --git a/packages/caliper-core/lib/monitor/monitor-process.js b/packages/caliper-core/lib/monitor/monitor-process.js index af2bb3d7e..75e5e3595 100644 --- a/packages/caliper-core/lib/monitor/monitor-process.js +++ b/packages/caliper-core/lib/monitor/monitor-process.js @@ -12,132 +12,27 @@ * limitations under the License. */ - 'use strict'; -const ps = require('ps-node'); -const usage = require('pidusage'); const MonitorInterface = require('./monitor-interface'); -const Util = require('../utils/caliper-utils.js'); -const logger = Util.getLogger('monitor-process.js'); - -/** - * Initialise a state object - * @return {JSON} state object - */ -function newStat() { - return { - mem_usage: [], - cpu_percent: [] - }; -} - -/** - * Construct the identity of the process - * @param {JSON} proc filter item of the process - * @return {String} identity - */ -function getId(proc) { - let id = proc.command; - if(proc.hasOwnProperty('arguments')) { - id += ' ' + proc.arguments; - } - - if(proc.hasOwnProperty('multiOutput')) { - id += '(' + proc.multiOutput + ')'; - } - else { - id += '(sum)'; - } - - return id; -} - - -/** -* Find processes according to the lookup filter -* @param {JSON} item lookup filter, must contains the 'command' element. Refer to https://www.npmjs.com/package/ps-node to learn more details. -* @return {Promise} array of pids of found processes -*/ -function findProcs(item) { - return new Promise((resolve, reject) => { - let pids = []; - ps.lookup(item, (err, resultList) => { - if (err) { - logger.error('failed looking the process up: ' + err); - } - else { - for(let i = 0 ; i < resultList.length ; i++) { - pids.push(resultList[i].pid); - } - } - resolve(pids); - }); - }); -} - -/** -* Get the memory and cpu usage of the specified process -* @param {String} pid the process's pid -* @return {Promise} JSON object as {cpu, memory} -*/ -function getProcUsage(pid) { - return new Promise((resolve, reject) => { - usage.stat(pid, (err, stat) => { - if(err) { - resolve({memory:0, cpu:0}); - } - else { - resolve(stat); - } - }); - }); -} - -/** -* Get the memory and cpu usage of multiple processes -* @param {Array} pids pids of specified processes -* @param {String} type = avg, return the average usage of all processes; = sum(default), return the summing usage of all processes -* @return {Promise} JSON object as {cpu, memory} -*/ -function getUsage(pids, type) { - return new Promise((resolve, reject) => { - let res = {memory: 0, cpu: 0}; - if(pids.length === 0) { - return resolve(res); - } +const MonitorUtilities = require('./monitor-utilities'); +const Util = require('../utils/caliper-utils'); +const Logger = Util.getLogger('monitor-process'); - let promises = pids.map((pid, idx) => { - return getProcUsage(pid); - }); +const ps = require('ps-node'); +const usage = require('pidusage'); - Promise.all(promises).then((stats) => { - for(let i = 0 ; i< stats.length ; i++) { - res.memory += stats[i].memory; - res.cpu += stats[i].cpu; - } - if(type === 'avg') { - res.memory /= stats.length; - res.cpu /= stats.length; - } - resolve(res); - }).catch((err) => { - logger.error('Exception encountered when fetching resource usage: ' + err); - resolve(res); - }); - }); -} /** * * Resource monitor for local processes */ class MonitorProcess extends MonitorInterface { /** * Constructor - * @param {JSON} filter lookup filter + * @param {JSON} monitorConfig Configuration object for the monitor * @param {*} interval resource fetching interval */ - constructor(filter, interval) { - super(filter, interval); + constructor(monitorConfig, interval) { + super(monitorConfig, interval); this.isReading = false; this.intervalObj = null; this.pids = {}; // pid history array @@ -152,99 +47,196 @@ class MonitorProcess extends MonitorInterface { } */ this.stats = {'time': []}; - this.filter = []; - for(let i = 0 ; i < filter.length ; i++) { - if(filter[i].hasOwnProperty('command')) { - let id = getId(filter[i]); - this.stats[id] = newStat(); - this.filter.push(filter[i]); + this.watchItems = []; + for(let i = 0 ; i < this.monitorConfig.length ; i++) { + if(this.monitorConfig[i].hasOwnProperty('command')) { + let id = this.getId(this.monitorConfig[i]); + this.stats[id] = this.newStat(); + this.watchItems.push(this.monitorConfig[i]); } } + } - + /** + * Initialise a state object + * @return {JSON} state object + */ + newStat() { + return { + mem_usage: [], + cpu_percent: [] + }; } /** - * Start the monitor - * @return {Promise} promise object + * Construct the identity of the process + * @param {JSON} proc filter item of the process + * @return {String} identity */ - start() { - let self = this; - /** - * Read statistics of watched items - */ - function readStats() { - if(self.isReading) { - return; - } - self.isReading = true; - - let promises = []; - self.filter.forEach((item) => { - promises.push(new Promise((resolve, reject) => { - // processes may be up/down during the monitoring, so should look for processes every time - findProcs(item).then((pids)=>{ - if(pids.length === 0) { - throw new Error('Not found any item'); - } - // record pids for later use (clear data) - for(let i = 0 ; i < pids.length ; ++i) { - self.pids[pids[i]] = 0; - } - // get usage for all processes - return getUsage(pids, item.multiOutput); - }).then((stat) => { - self.stats[getId(item)].mem_usage.push(stat.memory); - self.stats[getId(item)].cpu_percent.push(stat.cpu); - resolve(); - }).catch((err) => { - resolve(); - }); - })); + getId(proc) { + let id = proc.command; + if (proc.hasOwnProperty('arguments')) { + id += ' ' + proc.arguments; + } + + if (proc.hasOwnProperty('multiOutput')) { + id += '(' + proc.multiOutput + ')'; + } else { + id += '(sum)'; + } + + return id; + } + + + /** + * Find processes according to the lookup filter + * @param {JSON} item lookup filter, must contains the 'command' element. Refer to https://www.npmjs.com/package/ps-node to learn more details. + * @return {Promise} Array containing array of pids of found processes + */ + findProcesses(item) { + return new Promise((resolve, reject) => { + let pids = []; + ps.lookup(item, (err, resultList) => { + if (err) { + Logger.error('failed looking the process up: ' + err); + } else { + for (let i = 0 ; i < resultList.length ; i++) { + pids.push(resultList[i].pid); + } + } + resolve(pids); }); + }); + } + /** + * Get the memory and cpu usage of the specified process + * @param {String} pid the process's pid + * @return {JSON} JSON object as {cpu, memory} + * @async + */ + async getProcUsage(pid) { + return new Promise((resolve, reject) => { + usage.stat(pid, (error, stat) => { + if (error) { + resolve({memory:0, cpu:0}); + } else { + resolve(stat); + } + }); + }); + } - Promise.all(promises).then(() => { - self.isReading = false; - }).catch((err) => { - logger.error('Exception occurred when looking the process up: ' + err); + /** + * Get the memory and cpu usage of multiple processes + * @param {Array} pids pids of specified processes + * @param {String} type = avg, return the average usage of all processes; = sum(default), return the summing usage of all processes + * @return {JSON} JSON object as {cpu, memory} + * @async + */ + async getUsage(pids, type) { + try { + let res = {memory: 0, cpu: 0}; + if(pids.length === 0) { + return res; + } + + let promises = pids.map((pid, idx) => { + return this.getProcUsage(pid); }); + const stats = await Promise.all(promises); + + for(let i = 0 ; i< stats.length ; i++) { + res.memory += stats[i].memory; + res.cpu += stats[i].cpu; + } + if(type === 'avg') { + res.memory /= stats.length; + res.cpu /= stats.length; + } + return res; + } catch (error) { + Logger.warn(`Exception encountered when fetching resource usage of type ${type}`); + return {memory: 0, cpu: 0}; } - readStats(); - this.intervalObj = setInterval(readStats, this.interval); - return Promise.resolve(); + } + + /** + * Statistics read loop + * @async + */ + async readStats() { + Logger.debug('Entering readStats()'); + if (!this.isReading) { + try { + this.isReading = true; + for (let proc of this.watchItems) { + const pids = await this.findProcesses(proc); + if (!pids || pids.length === 0) { + // Does not exist + continue; + } + // record pids for later use (clear data) + for (let i = 0 ; i < pids.length ; ++i) { + this.pids[pids[i]] = 0; + } + // get usage for all processes + let name = this.getId(proc); + const stat = await this.getUsage(pids, proc.multiOutput); + this.stats[name].mem_usage.push(stat.memory); + this.stats[name].cpu_percent.push(stat.cpu); + } + } catch (error) { + Logger.error('Exception occurred when reading process statistics: ' + error); + } finally { + this.isReading = false; + } + } + Logger.debug('Exiting readStats()'); + } + + /** + * Start the monitor + * @async + */ + async start() { + await this.readStats(); + // Start interval monitor + const self = this; + this.intervalObj = setInterval(async () => { await self.readStats(); } , this.interval); + Logger.info(`Starting process monitor with update interval ${this.interval} ms`); } /** * Restart the monitor - * @return {Promise} promise object + * @async */ - restart() { + async restart() { clearInterval(this.intervalObj); - for(let key in this.stats) { - if(key === 'time') { + for (let key in this.stats) { + if (key === 'time') { this.stats[key] = []; - } - else { - for(let v in this.stats[key]) { + } else { + for (let v in this.stats[key]) { this.stats[key][v] = []; } } } - for(let key in this.pids) { + for (let key in this.pids) { usage.unmonitor(key); } this.pids = []; - return this.start(); + await this.start(); } /** * Stop the monitor - * @return {Promise} promise object + * @async */ - stop() { + async stop() { clearInterval(this.intervalObj); this.containers = []; this.stats = {'time': []}; @@ -254,41 +246,65 @@ class MonitorProcess extends MonitorInterface { } this.pids = []; - return Util.sleep(100); + await Util.sleep(100); } /** - * Get information of watched processes - * info = { - * key: lookup key of the process - * info: { - * TYPE: 'Process', - * NAME: name of the process - * } - * } - * @return {Array} array of processes' information + * Get a Map of result items + * @return {Map} Map of items to build results for, with default null entries */ - getPeers() { - let info = []; - for(let i in this.filter) { - let proc = this.filter[i]; - let name = getId(proc); - info.push({ - 'key' : name, - 'info' : { - 'TYPE' : 'Process', - 'NAME' : name - } - }); + getResultColumnMap() { + const columns = ['Type', 'Name', 'Memory(max)', 'Memory(avg)', 'CPU% (max)', 'CPU% (avg)']; + const resultMap = new Map(); + + for (const item of columns) { + resultMap.set(item, 'N/A'); } - return info; + // This is a Process Type, so set here + resultMap.set('Type', 'Process'); + return resultMap; + } + + /** + * Get statistics from the monitor in the form of an Array containing Map detailing key/value pairs + * @return {Map[]} an array of resource maps for watched containers + * @async + */ + async getStatistics() { + try { + const watchItemStats = []; + for (const watchItem of this.watchItems) { + const key = this.getId(watchItem); + + // retrieve stats for the key + let mem = this.getMemHistory(key); + let cpu = this.getCpuHistory(key); + let mem_stat = MonitorUtilities.getStatistics(mem); + let cpu_stat = MonitorUtilities.getStatistics(cpu); + + // Store in a Map + const watchItemStat = this.getResultColumnMap(); + watchItemStat.set('Name', key); + watchItemStat.set('Memory(max)', MonitorUtilities.byteNormalize(mem_stat.max)); + watchItemStat.set('Memory(avg)', MonitorUtilities.byteNormalize(mem_stat.avg)); + watchItemStat.set('CPU% (max)', cpu_stat.max.toFixed(2)); + watchItemStat.set('CPU% (avg)', cpu_stat.avg.toFixed(2)); + + // append return array + watchItemStats.push(watchItemStat); + } + return watchItemStats; + } catch (error) { + Logger.error('Failed to read monitoring data, ' + (error.stack ? error.stack : error)); + return []; + } } /** * Get history of memory usage * @param {String} key lookup key - * @return {Array} array of memory usage + * @return {number[]} array of memory usage */ getMemHistory(key) { // just to keep the same length as getCpuHistory @@ -298,7 +314,7 @@ class MonitorProcess extends MonitorInterface { /** * Get history of CPU usage * @param {String} key key of the container - * @return {Array} array of CPU usage + * @return {number[]} array of CPU usage */ getCpuHistory(key) { // the first element is an average from the starting time of the process @@ -306,24 +322,5 @@ class MonitorProcess extends MonitorInterface { return this.stats[key].cpu_percent.slice(1); } - /** - * Get history of network IO usage as {in, out} - * @param {String} key key of the container - * @return {Array} array of network IO usage - */ - getNetworkHistory(key) { - // not supported now return {'in': this.stats[key].netIO_rx, 'out':this.stats[key].netIO_tx}; - return {'in': [], 'out': []}; - } - - /** - * Get history of disc usage as {read, wrtie} - * @param {String} key key of the container - * @return {Array} array of disc usage - */ - getDiscHistory(key) { - // not supported now return {'in': this.stats[key].netIO_rx, 'out':this.stats[key].netIO_tx}; - return {'read': [], 'write': []}; - } } module.exports = MonitorProcess; diff --git a/packages/caliper-core/lib/monitor/monitor-prometheus.js b/packages/caliper-core/lib/monitor/monitor-prometheus.js new file mode 100644 index 000000000..498676b6f --- /dev/null +++ b/packages/caliper-core/lib/monitor/monitor-prometheus.js @@ -0,0 +1,172 @@ +/* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +'use strict'; + +const Util = require('../utils/caliper-utils.js'); + +const Logger = Util.getLogger('monitor-prometheus.js'); +const MonitorInterface = require('./monitor-interface'); +const PrometheusPushClient = require('../prometheus/prometheus-push-client'); +const PrometheusQueryClient = require('../prometheus/prometheus-query-client'); +const PrometheusQueryHelper = require('../prometheus/prometheus-query-helper'); + +/** + * Prometheus monitor implementation + */ +class PrometheusMonitor extends MonitorInterface { + + /** + * Constructor + * @param {JSON} monitorConfig Monitor config information + * @param {*} interval resource fetching interval + */ + constructor(monitorConfig, interval) { + super(monitorConfig, interval); + this.prometheusPushClient = new PrometheusPushClient(monitorConfig.pushUrl); + this.prometheusQueryClient = new PrometheusQueryClient(monitorConfig.url); + + // User defined options for monitoring + if (monitorConfig.hasOwnProperty('metrics')) { + // Might have an ignore list + if (monitorConfig.metrics.hasOwnProperty('ignore')) { + this.ignore = monitorConfig.metrics.ignore; + } else { + Logger.info('No monitor metrics `ignore` option specified, will provide statistics on all items retrieved by queries'); + } + + // Might have user specified queries to run + if (monitorConfig.metrics.hasOwnProperty('include')) { + this.include = monitorConfig.metrics.include; + } else { + Logger.info('No monitor metrics `include` options specified, unable to provide statistics on any resources'); + } + } else { + Logger.info('No monitor `metrics` specified, will not provide statistics on any resources'); + } + } + + /** + * Retrieve the push client + * @returns {PrometheusPushClient} the push client + */ + getPushClient(){ + return this.prometheusPushClient; + } + + /** + * Retrieve the query client + * @returns {PrometheusQueryClient} the query client + */ + getQueryClient(){ + return this.prometheusQueryClient; + } + + /** + * Retrieve the PushGateway URL from the monitor config + * @returns{String} the PushGateway URL + */ + getPushGatewayURL(){ + return this.monitorConfig.push_url; + } + + /** + * start monitoring - reset the initial query time index + * @async + */ + async start() { + this.startTime = Date.now()/1000; + } + + /** + * Stop the monitor - kill startTime + * @async + */ + async stop() { + this.startTime = undefined; + } + + /** + * restart monitoring - reset the initial query time index + * @async + */ + async restart() { + await this.start(); + } + + /** + * Get a Map of Prometheus query result items + * @param {string} query the prometheus query to be made + * @param {string} tag the short tag name for the query + * @return {Map} Map of items to build results for, with default null entries + */ + getResultColumnMapForQueryTag(query, tag) { + const resultMap = new Map(); + resultMap.set('Metric', tag); + resultMap.set('Prometheus Query', query); + resultMap.set('Name', 'N/A'); + return resultMap; + } + + /** + * Get statistics from Prometheus via queries that target the Prometheus server + * @returns {Map[]} Array of Maps detailing the resource utilization requests + * @async + */ + async getStatistics() { + this.endTime = Date.now()/1000; + + if (this.include) { + const resourceStats = []; + + for (const metricKey of Object.keys(this.include)) { + let newKey = true; + // Each metric is of the form + // Tag0: { + // query: 'the prometheus query to be made', + // statistic: 'the action to be taken on returned metrics' + // step: step size + // } + const params = this.include[metricKey]; + const queryString = PrometheusQueryHelper.buildStringRangeQuery(params.query, this.startTime, this.endTime, params.step); + const response = await this.prometheusQueryClient.getByEncodedUrl(queryString); + + // Retrieve base mapped statistics and coerce into correct format + const resultMap = PrometheusQueryHelper.extractStatisticFromRange(response, params.statistic, params.label); + + for (const [key, value] of resultMap.entries()) { + // Filter here + if (this.ignore.includes(key)) { + continue; + } else { + // Transfer into display array + const watchItemStat = newKey ? this.getResultColumnMapForQueryTag(params.query, metricKey) : this.getResultColumnMapForQueryTag('', ''); + watchItemStat.set('Name', key); + const multiplier = params.multiplier ? params.multiplier : 1; + watchItemStat.set('Value', (value*multiplier).toFixed(2)); + // Store + resourceStats.push(watchItemStat); + newKey = false; + } + } + } + return resourceStats; + } else { + Logger.debug('No include options specified for monitor - skipping action'); + } + } + +} + +module.exports = PrometheusMonitor; diff --git a/packages/caliper-core/lib/monitor/monitor-utilities.js b/packages/caliper-core/lib/monitor/monitor-utilities.js new file mode 100644 index 000000000..c10e2907c --- /dev/null +++ b/packages/caliper-core/lib/monitor/monitor-utilities.js @@ -0,0 +1,82 @@ +/* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + + +'use strict'; + +/** + * Static utility methods for monitors + */ +class MonitorUtilities { + + /** + * Cut down the string in case it's too long + * @param {String} data input string + * @return {String} normalized string + */ + static strNormalize(data) { + if(!data || typeof data !== 'string') { + return '-'; + } + + const maxLen = 30; + if(data.length <= maxLen) { + return data; + } + + return data.slice(0,25) + '...' + data.slice(-5); + } + + /** + * Normalize the value in byte + * @param {Number} data value in byte + * @return {String} value in string + */ + static byteNormalize(data) { + if(isNaN(data)) { + return '-'; + } + let kb = 1024; + let mb = kb * 1024; + let gb = mb * 1024; + if(data < kb) { + return data.toString() + 'B'; + } + else if(data < mb) { + return (data / kb).toFixed(1) + 'KB'; + } + else if(data < gb) { + return (data / mb).toFixed(1) + 'MB'; + } + else{ + return (data / gb).toFixed(1) + 'GB'; + } + } + + /** + * Get statistics(maximum, minimum, summation, average) of a number array + * @param {Array} arr array of numbers + * @return {JSON} JSON object as {max, min, total, avg} + */ + static getStatistics(arr) { + if(arr.length === 0) { + return {max : NaN, min : NaN, total : NaN, avg : NaN}; + } else { + const sum = arr.reduce((x, y) => x + y); + return {max : Math.max(...arr), min : Math.min(...arr), total : sum, avg : sum/arr.length}; + } + } +} + +module.exports = MonitorUtilities; diff --git a/packages/caliper-core/lib/monitor/monitor.js b/packages/caliper-core/lib/monitor/monitor.js deleted file mode 100644 index 132ee1114..000000000 --- a/packages/caliper-core/lib/monitor/monitor.js +++ /dev/null @@ -1,508 +0,0 @@ -/* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - - -'use strict'; - -const table = require('table'); -const Util = require('../utils/caliper-utils'); -const logger= Util.getLogger('monitor.js'); - -/** -* Get statistics(maximum, minimum, summation, average) of a number array -* @param {Array} arr array of numbers -* @return {JSON} JSON object as {max, min, total, avg} -*/ -function getStatistics(arr) { - if(arr.length === 0) { - return {max : NaN, min : NaN, total : NaN, avg : NaN}; - } - - let max = arr[0], min = arr[0], total = arr[0]; - for(let i = 1 ; i< arr.length ; i++) { - let value = arr[i]; - if(value > max) { - max = value; - } - if(value < min) { - min = value; - } - total += value; - } - - return {max : max, min : min, total : total, avg : total/arr.length}; -} - -/** -* Normalize the value in byte -* @param {Number} data value in byte -* @return {String} value in string -*/ -function byteNormalize(data) { - if(isNaN(data)) { - return '-'; - } - let kb = 1024; - let mb = kb * 1024; - let gb = mb * 1024; - if(data < kb) { - return data.toString() + 'B'; - } - else if(data < mb) { - return (data / kb).toFixed(1) + 'KB'; - } - else if(data < gb) { - return (data / mb).toFixed(1) + 'MB'; - } - else{ - return (data / gb).toFixed(1) + 'GB'; - } -} - -/** -* Cut down the string in case it's too long -* @param {String} data input string -* @return {String} normalized string -*/ -function strNormalize(data) { - if(!data || typeof data !== 'string') { - return '-'; - } - - const maxLen = 30; - if(data.length <= maxLen) { - return data; - } - - let newstr = data.slice(0,25) + '...' + data.slice(-5); - return newstr; -} - -/** - * Monitor class, containing operations to watch resource consumption of specific destinations - */ -class Monitor { - /** - * Constructor - * @param {String} configPath path of the configuration file - */ - constructor(configPath) { - this.configPath = configPath; - this.started = false; - this.peers = []; - this.monitors = []; - } - - /** - * start the monitor - * @return {Promise} promise object - */ - start() { - //const config = require(this.configPath); - const config = Util.parseYaml(this.configPath); - const m = config.monitor; - if(typeof m === 'undefined') { - return Promise.reject(new Error('Failed to find monitor in config file')); - } - - let monitorTypes = m.type; - if(typeof monitorTypes === 'undefined') { - return Promise.reject(new Error('Failed to find monitor type in config file')); - } - if(!Array.isArray(monitorTypes)) { - monitorTypes = [monitorTypes]; - } - - let p; - if(this.started === true) { - p = this.stop(); - } - else { - p = Promise.resolve(); - } - - return p.then(() => { - let promises = []; - monitorTypes.forEach( (type) => { - promises.push(new Promise((resolve, reject) => { - let promise = null; - if(type === 'docker') { // monitor for local docker containers - promise = this._startDockerMonitor(m.docker, m.interval); - } - else if(type === 'process') { - promise = this._startProcessMonitor(m.process, m.interval); - } - else { - logger.error('undefined monitor type: ' + type); - return resolve(); - } - promise.then((monitor)=>{ - this.monitors.push(monitor); - resolve(); - }).catch((err)=>{ - logger.error('start monitor ' + type + ' failed: ' + err); - resolve(); // always return resolve for Promsie.all - }); - })); - }); - return Promise.all(promises); - }).then(() => { - this.started = true; - return Promise.resolve(); - }).catch((err) => { - return Promise.reject(err); - }); - } - - /** - * stop the monitor - * @return {Promise} promise object - */ - stop() { - if( this.monitors.length > 0 && this.started === true) { - let promises = []; - this.monitors.forEach((monitor)=>{ - promises.push(new Promise((resolve, reject) => { - monitor.stop().then(() => { - resolve(); - }).catch((err) => { - logger.error('stop monitor failed: ' + err); - resolve(); - }); - })); - }); - return Promise.all(promises).then(()=>{ - this.monitors = []; - this.peers = []; - this.started = false; - return Promise.resolve(); - }).catch((err)=>{ - logger.error('stop monitor failed: ' + err); - this.monitors = []; - this.peers = []; - this.started = false; - return Promise.resolve(); - }); - } - - return Promise.resolve(); - } - - /** - * restart the monitor, all data recorded internally will be cleared - * @return {Promise} promise object - */ - restart() { - if(this.monitors.length > 0 && this.started === true){ - this._readDefaultStats(false); - let promises = []; - this.monitors.forEach((monitor)=>{ - promises.push(new Promise((resolve, reject) => { - monitor.restart().then(() => { - resolve(); - }).catch((err) => { - logger.error('restart monitor failed: ' + err); - resolve(); - }); - })); - }); - return Promise.all(promises); - } - - return this.start(); - } - - /** - * Get the default statistics table - * @return {Array} statistics table - */ - getDefaultStats() { - try { - this._readDefaultStats(true); - - if(this.peers === null || this.peers.length === 0) { - logger.error('Failed to read monitoring data'); - return []; - } - - let defaultTable = []; - let tableHead = []; - for(let i in this.peers[0].info) { - tableHead.push(i); - } - let historyItems = this._getDefaultItems(); - tableHead.push.apply(tableHead, historyItems); - - defaultTable.push(tableHead); - for(let i in this.peers){ - let row = []; - for(let j in this.peers[i].info) { - row.push(strNormalize(this.peers[i].info[j])); - } - - let historyValues = this._getLastHistoryValues(historyItems, i); - row.push.apply(row, historyValues); - defaultTable.push(row); - } - - return defaultTable; - } - catch(err) { - logger.error('Failed to read monitoring data, ' + (err.stack ? err.stack : err)); - return []; - } - } - - /** - * Print the maximum values of all watched items - */ - printMaxStats() { - try { - this._readDefaultStats(true); - - if(this.peers === null || this.peers.length === 0) { - logger.error('Failed to read monitoring data'); - return; - } - - let defaultTable = []; - let tableHead = []; - for(let i in this.peers[0].info) { - tableHead.push(i); - } - let historyItems = this._getMaxItems(); - tableHead.push.apply(tableHead, historyItems); - - defaultTable.push(tableHead); - for(let i in this.peers){ - let row = []; - for(let j in this.peers[i].info) { - row.push(strNormalize(this.peers[i].info[j])); - } - - let historyValues = this._getMaxHistoryValues(historyItems, i); - row.push.apply(row, historyValues); - defaultTable.push(row); - } - - let t = table.table(defaultTable, {border: table.getBorderCharacters('ramac')}); - logger.info('\n ### resource stats (maximum) ###'); - logger.info('\n' + t); - } - catch(err) { - logger.error('Failed to read monitoring data, ' + (err.stack ? err.stack : err)); - } - } - - /** - * pseudo private functions - */ - - /** - * read current statistics from monitor object and push the data into peers.history object - * the history data will not be cleared until stop() is called, in other words, calling restart will not vanish the data - * @param {Boolean} tmp =true, the data should only be stored in history temporarily - */ - _readDefaultStats(tmp) { - if (this.peers.length === 0) { - for(let i = 0 ; i < this.monitors.length ; i++) - { - let newPeers = this.monitors[i].getPeers(); - newPeers.forEach((peer) => { - peer.history = { - 'Memory(max)' : [], - 'Memory(avg)' : [], - 'CPU(max)' : [], - 'CPU(avg)' : [], - 'Traffic In' : [], - 'Traffic Out' : [], - 'Disc Read' : [], - 'Disc Write' : [] - }; - peer.isLastTmp = false; - peer.monitor = this.monitors[i]; - this.peers.push(peer); - }); - } - } - - this.peers.forEach((peer) => { - let key = peer.key; - let mem = peer.monitor.getMemHistory(key); - let cpu = peer.monitor.getCpuHistory(key); - let net = peer.monitor.getNetworkHistory(key); - let disc = peer.monitor.getDiscHistory(key); - let mem_stat = getStatistics(mem); - let cpu_stat = getStatistics(cpu); - if(peer.isLastTmp) { - let lastIdx = peer.history['Memory(max)'].length - 1; - peer.history['Memory(max)'][lastIdx] = mem_stat.max; - peer.history['Memory(avg)'][lastIdx] = mem_stat.avg; - peer.history['CPU(max)'][lastIdx] = cpu_stat.max; - peer.history['CPU(avg)'][lastIdx] = cpu_stat.avg; - peer.history['Traffic In'][lastIdx] = net.in[net.in.length-1] - net.in[0]; - peer.history['Traffic Out'][lastIdx] = net.out[net.out.length-1] - net.out[0]; - peer.history['Disc Write'][lastIdx] = disc.write[disc.write.length-1] - disc.write[0]; - peer.history['Disc Read'][lastIdx] = disc.read[disc.read.length-1] - disc.read[0]; - } - else { - peer.history['Memory(max)'].push(mem_stat.max); - peer.history['Memory(avg)'].push(mem_stat.avg); - peer.history['CPU(max)'].push(cpu_stat.max); - peer.history['CPU(avg)'].push(cpu_stat.avg); - peer.history['Traffic In'].push(net.in[net.in.length-1] - net.in[0]); - peer.history['Traffic Out'].push(net.out[net.out.length-1] - net.out[0]); - peer.history['Disc Write'].push(disc.write[disc.write.length-1] - disc.write[0]); - peer.history['Disc Read'].push(disc.read[disc.read.length-1] - disc.read[0]); - } - peer.isLastTmp = tmp; - }); - } - - /** - * Get names of default historical data - * @return {Array} array of names - */ - _getDefaultItems() { - let items = []; - for(let key in this.peers[0].history) { - if(this.peers[0].history.hasOwnProperty(key)) { - items.push(key); - } - } - return items; - } - - /** - * Get names of maximum related historical data - * @return {Array} array of names - */ - _getMaxItems() { - return ['Memory(max)', 'CPU(max)', 'Traffic In','Traffic Out', 'Disc Read', 'Disc Write']; - } - - - /** - * Get last values of specific historical data - * @param {Array} items names of the lookup items - * @param {Number} idx index of the lookup object - * @return {Array} normalized string values - */ - _getLastHistoryValues(items, idx) { - let values = []; - for(let i = 0 ; i < items.length ; i++) { - let key = items[i]; - if (!this.peers[idx].history.hasOwnProperty(key)) { - logger.warn('could not find history object named ' + key); - values.push('-'); - continue; - } - let length = this.peers[idx].history[key].length; - if(length === 0) { - logger.warn('could not find history data of ' + key); - values.push('-'); - continue; - } - let value = this.peers[idx].history[key][length - 1]; - if(key.indexOf('Memory') === 0 || key.indexOf('Traffic') === 0 || key.indexOf('Disc') === 0) { - values.push(byteNormalize(value)); - } - else if(key.indexOf('CPU') === 0) { - values.push(value.toFixed(2) + '%'); - } - else{ - values.push(value.toString()); - } - } - - return values; - } - - /** - * get the maximum value of specific historical data - * @param {Array} items names of the lookup items - * @param {Number} idx index of the lookup object - * @return {Array} array of normalized strings - */ - _getMaxHistoryValues(items, idx) { - let values = []; - for(let i = 0 ; i < items.length ; i++) { - let key = items[i]; - if (!this.peers[idx].history.hasOwnProperty(key)) { - logger.warn('could not find history object named ' + key); - values.push('-'); - continue; - } - let length = this.peers[idx].history[key].length; - if(length === 0) { - logger.warn('could not find history data of ' + key); - values.push('-'); - continue; - } - let stats = getStatistics(this.peers[idx].history[key]); - if(key.indexOf('Memory') === 0 || key.indexOf('Traffic') === 0 || key.indexOf('Disc') === 0) { - values.push(byteNormalize(stats.max)); - } - else if(key.indexOf('CPU') === 0) { - values.push(stats.max.toFixed(2) + '%'); - } - else{ - values.push(stats.max.toString()); - } - } - - return values; - } - - /** - * Start a monitor for docker containers - * @param {JSON} args lookup filter - * @param {Number} interval read interval, in second - * @return {Promise} promise object - */ - _startDockerMonitor(args, interval) { - if(typeof args === 'undefined') { - args = {'name': ['all']}; - } - if(typeof interval === 'undefined') { - interval = 1; - } - - let DockerMonitor = require('./monitor-docker.js'); - let monitor = new DockerMonitor(args, interval); - return monitor.start().then(()=>{ - return Promise.resolve(monitor); - }).catch((err)=>{ - return Promise.reject(err); - }); - } - - /** - * Start a monitor for local processes - * @param {JSON} args lookup filter - * @param {Number} interval read interval, in second - * @return {Promise} promise object - */ - _startProcessMonitor(args, interval) { - let ProcessMonitor = require('./monitor-process.js'); - let monitor = new ProcessMonitor(args, interval); - return monitor.start().then(()=>{ - return Promise.resolve(monitor); - }).catch((err)=>{ - return Promise.reject(err); - }); - } -} - -module.exports = Monitor; diff --git a/packages/caliper-core/lib/prometheus/prometheus-push-client.js b/packages/caliper-core/lib/prometheus/prometheus-push-client.js new file mode 100644 index 000000000..338b108f6 --- /dev/null +++ b/packages/caliper-core/lib/prometheus/prometheus-push-client.js @@ -0,0 +1,146 @@ +/* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +'use strict'; + +const url = require('url'); +const http = require('http'); +const https = require('https'); + +const Logger = require('../utils/caliper-utils').getLogger('prometheus-push-client'); + +const BASE = 'metrics/job/caliper/'; +const POST = 'POST'; +const DELETE = 'DELETE'; + +/** + * PrometheusClient - client communication with Prometheus metrics through the Push Gateway + */ +class PrometheusPushClient { + + /** + * Constructor for client + * @param {String} gatewayURL the push gateway URL + */ + constructor(gatewayURL) { + this.gatewayURL = gatewayURL; + } + + /** + * Check if gateway has been set + * @returns {Boolean} true if gateway set, otherwise false + */ + gatewaySet(){ + const isSet = this.gatewayURL ? true : false; + return isSet; + } + + /** + * Set the gateway + * @param {String} gatewayURL the push gateway URL + */ + setGateway(gatewayURL) { + this.gatewayURL = gatewayURL; + } + + /** + * Configure the target for the push + * @param {String} testLabel the benchmark test name to store under + * @param {String} testRound the test round to store under + * @param {String} clientId the clientId to store under + */ + configureTarget(testLabel, testRound, clientId) { + const testPath = `instance/${testLabel}/round/${testRound.toString()}/client/${clientId.toString()}`; + const target = url.resolve(this.gatewayURL, BASE + testPath); + this.requestParams = url.parse(target); + this.httpModule = this.isHttps(this.requestParams.href) ? https : http; + Logger.debug(`Prometheus push client configured to target ${this.requestParams.href}`); + } + + /** + * Push a message to the Prometheus gateway + * @param {String} key the key to store the information + * @param {String} value the value to persist + * @param {String[]} tags the tags to use when persisting + */ + push(key, value, tags) { + let body; + if (tags) { + body = `${key}{${tags.join(',')}} ${value}`; + } else { + body = `${key} ${value}`; + } + this.useGateway(POST, body); + } + + /** + * Delete everything under the path within the PushGateway for the current configuration + */ + delete(){ + this.useGateway(DELETE, null); + } + + /** + * Send message on gateway + * @param {String} method the method type [POST | DELETE] + * @param {String} body the body to send + */ + useGateway(method, body) { + Logger.debug(`Prometheus client sending body ${body} to target ${this.requestParams.href}`); + // Convert body to binary, the newline is important + body = new Buffer(body + '\n', 'binary'); + + // Assign request options + const options = Object.assign(this.requestParams, { + method, + headers: { + 'Content-Type': 'application/octet-stream', + 'Content-Length': body.length + } + }); + + // Make the request + const req = this.httpModule.request(options, res => { + let body = ''; + res.setEncoding('utf8'); + res.on('data', chunk => { + body += chunk; + }); + res.on('end', () => { + if(body) { + Logger.info('PushGateway Response: ' + body); + } + }); + }); + req.on('error', err => { + Logger.error(err); + }); + + // send and end + req.write(body); + req.end(); + } + + /** + * Check if we are using http or https + * @param {*} href the passed Href + * @returns {Boolean} true if https + */ + isHttps(href) { + return href.search(/^https/) !== -1; + } + +} + +module.exports = PrometheusPushClient; diff --git a/packages/caliper-core/lib/prometheus/prometheus-query-client.js b/packages/caliper-core/lib/prometheus/prometheus-query-client.js new file mode 100644 index 000000000..f2130eea7 --- /dev/null +++ b/packages/caliper-core/lib/prometheus/prometheus-query-client.js @@ -0,0 +1,172 @@ +/* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +'use strict'; + +const url = require('url'); +const http = require('http'); +const https = require('https'); + +const Logger = require('../utils/caliper-utils').getLogger('prometheus-query-client'); + +const NULL = '/api/v1/'; +const RANGE = '/api/v1/query_range'; +const SINGLE = '/api/v1/query'; + +/** + * PrometheusQueryClient for use with querying the Prometheus server to retrieve metrics + */ +class PrometheusQueryClient { + + /** + * Constructor for client + * @param {String} prometheusUrl the url of the Prometheus gateway + */ + constructor(prometheusUrl) { + this.prometheusUrl = prometheusUrl; + } + + /** + * Retrieve target parameters for the query + * @param {String} type query type + * @param {String} query the unique query append + * @returns {Object} request parameters and the http module required for the http request + */ + retrieveTargetParameters(type, query) { + const target = url.resolve(this.prometheusUrl, type + query); + const requestParams = url.parse(target); + const httpModule = this.isHttps(requestParams.href) ? https : http; + return { + requestParams, + httpModule + }; + } + + /** + * Issue a range query to the Prometheus server + * @param {String} queryString string query to make + * @param {number} startTime start time index for the query, in seconds since epoch (ie Date.now()/1000) + * @param {number} endTime end time index for the query, in seconds since epoch (ie Date.now()/1000) + * @param {number} step step size for query, defaults if not provided + * @returns {JSON} the result of the query + * @async + */ + async rangeQuery(queryString, startTime, endTime, step = 1) { + const query = '?query=' + queryString + '&start=' + startTime + '&end=' + endTime + '&step=' + step; + Logger.debug('Issuing range query: ', query); + + const targetParams = this.retrieveTargetParameters(RANGE, query); + return await this.retrieveResponse(targetParams); + } + + /** + * Issue a query to the Prometheus server + * @param {String} queryString string query to make + * @param {number} timePoint time index for the query, in seconds since epoch (ie Date.now()/1000) + * @returns {JSON} the result of the query + * @async + */ + async query(queryString, timePoint) { + const query = '?query=' + queryString + '&time=' + timePoint; + Logger.debug('Issuing query: ', query); + const targetParams = this.retrieveTargetParameters(SINGLE, query); + return await this.retrieveResponse(targetParams); + } + + /** + * URL encode a passed string and issue a `get` request against the v1 api + * @param {String} getString the string to URL encode and use as a get request + * @returns {JSON} the result of the get query + * @async + */ + async getByEncodedUrl(getString) { + Logger.debug('Performing get with url encoded string: ', getString); + const targetParams = this.retrieveTargetParameters(NULL, getString); + return await this.retrieveResponse(targetParams); + } + + /** + * Retrieve a response object by making the query + * @param {Object} targetParams parameters and the http module required for the http request + * @returns {JSON} the result of the query + * @async + */ + async retrieveResponse(targetParams){ + try { + const resp = await this.doRequest(targetParams); + const response = JSON.parse(resp); + if (response.status.localeCompare('success') === 0 ) { + return response; + } else { + Logger.error(`Prometheus query to ${this.requestParams.href} failed`); + return null; + } + } catch (error) { + Logger.error('Query error: ', error); + throw error; + } + } + + + /** + * Perform the request and return it in a promise + * @param {Object} targetParams parameters and the http module required for the http request + * @returns {Promise} Promise + */ + doRequest(targetParams) { + return new Promise ((resolve, reject) => { + + // Assign request options + const options = Object.assign(targetParams.requestParams, { + method: 'GET', + headers: { + 'Content-Type': 'application/json' + } + }); + + // Make the request + const req = targetParams.httpModule.request(options, res => { + let body = ''; + res.setEncoding('utf8'); + res.on('data', chunk => { + body += chunk; + }); + res.on('end', () => { + if(body) { + resolve(body); + } else { + resolve(); + } + }); + res.on('error', err => { + Logger.error(err); + reject(err); + }); + }); + + req.end(); + }); + } + + /** + * Check if we are using http or https + * @param {*} href the passed Href + * @returns {Boolean} true if https + */ + isHttps(href) { + return href.search(/^https/) !== -1; + } +} + +module.exports = PrometheusQueryClient; diff --git a/packages/caliper-core/lib/prometheus/prometheus-query-helper.js b/packages/caliper-core/lib/prometheus/prometheus-query-helper.js new file mode 100644 index 000000000..064eb33d8 --- /dev/null +++ b/packages/caliper-core/lib/prometheus/prometheus-query-helper.js @@ -0,0 +1,191 @@ +/* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +'use strict'; + +const Logger = require('../utils/caliper-utils').getLogger('prometheus-query-helper'); + +/** + * PrometheusQueryHelper - static helper functions used to help with return responses from Prometheus queries + */ +class PrometheusQueryHelper { + + /** + * Build a string range query suitable for querying the Prometheus server + * @param {String} query the query to modify with start and end times + * @param {number} startTime the start time in seconds for the range query + * @param {number} endTime the end time in seconds for the range query + * @param {number} step the step size to use + * @returns {String} the string query to use + */ + static buildStringRangeQuery(query, startTime, endTime, step) { + // Anything that is within `{ }` must be URI encoded (including braces) + + const myRegexp = /({.*})/; + let match = myRegexp.exec(query); + while (match !== null) { + query = query.replace(myRegexp, encodeURIComponent(match[0])); + match = myRegexp.exec(query); + } + + const builtQuery = 'query_range?query=' + query + '&start=' + startTime + '&end=' + endTime + '&step=' + step; + return builtQuery; + } + + /** + * Extract a single numeric value from a Prometheus query response. A Prometheus query response may be a + * vector or a matrix. + * @param {JSON} response the JSON response from the prometheus server + * @param {boolean} isNumeric boolean to indicate if value should be cast to float (true) or not (false) + * @returns {Map | value} Either a map of values or single value depending on if the passed response is a matrix or vector + */ + static extractFirstValueFromQueryResponse(response, isNumeric = true) { + + switch(response.data.resultType){ + case 'vector': + // should have a result entry containing a 'value' field (not 'values') + if (response.data.result && response.data.result[0] && response.data.result[0].hasOwnProperty('value')) { + // value = [timeIndex, value] + const val = response.data.result[0].value[1]; + if (isNumeric) { + return parseFloat(val); + } else { + return val; + } + } else { + if (response.data.result && response.data.result[0]) { + Logger.error(`Empty or invalid response ${JSON.stringify(response)} passed for single value extraction`); + } + return '-'; + } + case 'matrix': + { + // We need to look at each matrix item and return a Map that details the name:value pairing for each entry + // result array contains JSON objects that are: + // { + // metric: { name: myName}, + // values: [[timeIndex, value], [timeIndex, value], ..., [timeIndex, value]]} + // } + const valuesMap = new Map(); + for (let result of response.data.result) { + const name = result.metric.name; + const value = result.values[0][1]; + if (isNumeric) { + valuesMap.set(name, parseFloat(value)); + } else { + valuesMap.set(name, value); + } + } + return valuesMap; + } + default: + throw new Error(`Unknown or missing result type: ${response.data.resultType}`); + } + } + + /** + * Extract a statistical value from a Prometheus range query response + * @param {JSON} response the JSON response from the prometheus server + * @param {String} statType the type of statistic to retrieve from data range + * @param {String} label the statistic label of interest + * @returns {number} the statistical value in the range query results + */ + static extractStatisticFromRange(response, statType, label){ + + switch(response.data.resultType){ + case 'vector': + // should have a result entry containing a 'value' field (not 'values') + if (response.data.result && response.data.result[0].hasOwnProperty('value')) { + Logger.error(`Invalid response ${JSON.stringify(response)} passed for single value extraction`); + } + break; + case 'matrix': + { + // We need to look at each matrix item and return a Map that details the name:value pairing for each entry + // result array contains JSON objects that are: + // { + // metric: { name: myName, otherName: anotherName, label: nameOfInterest }, + // values: [[timeIndex, value], [timeIndex, value], ..., [timeIndex, value]]} + // } + const valuesMap = new Map(); + for (let result of response.data.result) { + const name = (result.metric && result.metric[label]) ? result.metric[label] : 'unknown'; + const series = result.values ? result.values : []; + const values = this.extractValuesFromTimeSeries(series, true); + try { + const stat = this.retrieveStatisticFromArray(values, statType); + if (isNaN(stat) || stat === Infinity || stat === -Infinity){ + valuesMap.set(name, '-'); + } else { + valuesMap.set(name, stat); + } + } catch (error) { + Logger.warn(`Unable to perform Math operation, with error ${error}`); + valuesMap.set(name, '-'); + } + } + return valuesMap; + } + default: + throw new Error(`Unknown or missing result type: ${response.data.resultType}`); + } + } + + /** + * Perform a basic statistical operation over an array + * @param {Array} values array of values to perform the operation on + * @param {String} statType type of statistical operation + * @returns {number} result from statistical operation + */ + static retrieveStatisticFromArray(values, statType) { + switch(statType) { + case 'max': + return Math.max(...values); + case 'min': + return Math.min(...values); + case 'avg': + { + if (values.length>1) { + const sum = values.reduce((x, y) => x + y); + return sum / values.length; + } else { + return values[0]; + } + } + default: + Logger.error(`Unknown stat type passed: ${statType}`); + throw new Error(`Unknown stat type passed: ${statType}`); + } + } + + /** + * Extract values from time series data + * @param {Array} series Array of the form [ [timeIndex, value], [], ..., [] ] + * @param {boolean} isNumeric boolean to indicate if value should be cast to float (true) or not (false) + * @returns {Array} one dimensional array of values + */ + static extractValuesFromTimeSeries(series, isNumeric){ + const values = []; + for (const element of series) { + if (isNumeric) { + values.push(parseFloat(element[1])); + } else { + values.push(element[1]); + } + } + return values; + } +} + +module.exports = PrometheusQueryHelper; diff --git a/packages/caliper-core/lib/rate-control/compositeRate.js b/packages/caliper-core/lib/rate-control/compositeRate.js index ee17258c4..e8f246af7 100644 --- a/packages/caliper-core/lib/rate-control/compositeRate.js +++ b/packages/caliper-core/lib/rate-control/compositeRate.js @@ -209,7 +209,7 @@ class CompositeRateController extends RateInterface{ * @param {number} msg.txDuration The length of the round in SECONDS. * @param {number} msg.totalClients The number of clients executing the round. * @param {number} msg.clients The number of clients executing the round. - * @param {object} msg.clientargs Arguments for the client. + * @param {object} msg.clientArgs Arguments for the client. * @param {number} msg.clientIdx The 0-based index of the current client. * @param {number} msg.roundIdx The 1-based index of the current round. * @async diff --git a/packages/caliper-core/lib/rate-control/fixedFeedbackRate.js b/packages/caliper-core/lib/rate-control/fixedFeedbackRate.js index 416468e7a..9bd95eb91 100644 --- a/packages/caliper-core/lib/rate-control/fixedFeedbackRate.js +++ b/packages/caliper-core/lib/rate-control/fixedFeedbackRate.js @@ -48,7 +48,7 @@ class FixedFeedbackRateController extends RateInterface{ * @param {number} msg.txDuration The length of the round in SECONDS. * @param {number} msg.totalClients The number of clients executing the round. * @param {number} msg.clients The number of clients executing the round. - * @param {object} msg.clientargs Arguments for the client. + * @param {object} msg.clientArgs Arguments for the client. * @param {number} msg.clientIdx The 0-based index of the current client. * @param {number} msg.roundIdx The 1-based index of the current round. * @@ -96,7 +96,7 @@ class FixedFeedbackRateController extends RateInterface{ return; } // Determines the sleep time for waiting until - // successful transactions occure. + // successful transactions occur if(resultStats.length > 1 && resultStats[1].succ === 0) { this.zero_succ_count++; for(let i = 30; i > 0; --i) { diff --git a/packages/caliper-core/lib/rate-control/fixedRate.js b/packages/caliper-core/lib/rate-control/fixedRate.js index 2cdfef90b..988eb58d7 100644 --- a/packages/caliper-core/lib/rate-control/fixedRate.js +++ b/packages/caliper-core/lib/rate-control/fixedRate.js @@ -50,7 +50,7 @@ class FixedRate extends RateInterface { * @param {number} msg.txDuration The length of the round in SECONDS. * @param {number} msg.totalClients The number of clients executing the round. * @param {number} msg.clients The number of clients executing the round. - * @param {object} msg.clientargs Arguments for the client. + * @param {object} msg.clientArgs Arguments for the client. * @param {number} msg.clientIdx The 0-based index of the current client. * @param {number} msg.roundIdx The 1-based index of the current round. * @async diff --git a/packages/caliper-core/lib/rate-control/linearRate.js b/packages/caliper-core/lib/rate-control/linearRate.js index 216ddf960..0ff8e54a7 100644 --- a/packages/caliper-core/lib/rate-control/linearRate.js +++ b/packages/caliper-core/lib/rate-control/linearRate.js @@ -71,7 +71,7 @@ class LinearRateController extends RateInterface{ * @param {number} msg.txDuration The length of the round in SECONDS. * @param {number} msg.totalClients The number of clients executing the round. * @param {number} msg.clients The number of clients executing the round. - * @param {object} msg.clientargs Arguments for the client. + * @param {object} msg.clientArgs Arguments for the client. * @param {number} msg.clientIdx The 0-based index of the current client. * @param {number} msg.roundIdx The 1-based index of the current round. * @async diff --git a/packages/caliper-core/lib/rate-control/noRate.js b/packages/caliper-core/lib/rate-control/noRate.js index 801964f11..d8599ebb6 100644 --- a/packages/caliper-core/lib/rate-control/noRate.js +++ b/packages/caliper-core/lib/rate-control/noRate.js @@ -50,7 +50,7 @@ class NoRateController extends RateInterface{ * @param {number} msg.txDuration The length of the round in SECONDS. * @param {number} msg.totalClients The number of clients executing the round. * @param {number} msg.clients The number of clients executing the round. - * @param {object} msg.clientargs Arguments for the client. + * @param {object} msg.clientArgs Arguments for the client. * @param {number} msg.clientIdx The 0-based index of the current client. * @param {number} msg.roundIdx The 1-based index of the current round. * @async diff --git a/packages/caliper-core/lib/rate-control/rateControl.js b/packages/caliper-core/lib/rate-control/rateControl.js index b90d0a770..06278e842 100644 --- a/packages/caliper-core/lib/rate-control/rateControl.js +++ b/packages/caliper-core/lib/rate-control/rateControl.js @@ -14,7 +14,7 @@ 'use strict'; const CaliperUtils = require('../utils/caliper-utils'); -let logger = CaliperUtils.getLogger('rateControl.js'); +const logger = CaliperUtils.getLogger('rateControl.js'); const builtInControllers = new Map([ ['fixed-rate', './fixedRate.js'], @@ -27,7 +27,7 @@ const builtInControllers = new Map([ ['fixed-feedback-rate', './fixedFeedbackRate.js'] ]); -let RateControl = class { +const RateControl = class { /** * Instantiates the proxy rate controller and creates the configured rate controller behind it. @@ -65,7 +65,7 @@ let RateControl = class { * @param {number} msg.txDuration The length of the round in SECONDS. * @param {number} msg.totalClients The number of clients executing the round. * @param {number} msg.clients The number of clients executing the round. - * @param {object} msg.clientargs Arguments for the client. + * @param {object} msg.clientArgs Arguments for the client. * @param {number} msg.clientIdx The 0-based index of the current client. * @param {number} msg.roundIdx The 1-based index of the current round. * @async diff --git a/packages/caliper-core/lib/rate-control/rateInterface.js b/packages/caliper-core/lib/rate-control/rateInterface.js index d0f011bbf..801cbb6b5 100644 --- a/packages/caliper-core/lib/rate-control/rateInterface.js +++ b/packages/caliper-core/lib/rate-control/rateInterface.js @@ -43,7 +43,7 @@ class RateInterface { * @param {number} msg.txDuration The length of the round in SECONDS. * @param {number} msg.totalClients The number of clients executing the round. * @param {number} msg.clients The number of clients executing the round. - * @param {object} msg.clientargs Arguments for the client. + * @param {object} msg.clientArgs Arguments for the client. * @param {number} msg.clientIdx The 0-based index of the current client. * @param {number} msg.roundIdx The 1-based index of the current round. * @async diff --git a/packages/caliper-core/lib/rate-control/recordRate.js b/packages/caliper-core/lib/rate-control/recordRate.js index 30f7ff731..2f26551d9 100644 --- a/packages/caliper-core/lib/rate-control/recordRate.js +++ b/packages/caliper-core/lib/rate-control/recordRate.js @@ -131,7 +131,7 @@ class RecordRateController extends RateInterface{ * @param {number} msg.txDuration The length of the round in SECONDS. * @param {number} msg.totalClients The number of clients executing the round. * @param {number} msg.clients The number of clients executing the round. - * @param {object} msg.clientargs Arguments for the client. + * @param {object} msg.clientArgs Arguments for the client. * @param {number} msg.clientIdx The 0-based index of the current client. * @param {number} msg.roundIdx The 1-based index of the current round. * @async @@ -194,7 +194,7 @@ class RecordRateController extends RateInterface{ logger.debug(`Recorded Tx submission times for Client#${this.clientIdx} in Round#${this.roundIdx} to ${this.pathTemplate}`); } } catch (err) { - logger.error(`An error occured while writing records to ${this.pathTemplate}: ${err.stack ? err.stack : err}`); + logger.error(`An error occurred while writing records to ${this.pathTemplate}: ${err.stack ? err.stack : err}`); } } } diff --git a/packages/caliper-core/lib/rate-control/replayRate.js b/packages/caliper-core/lib/rate-control/replayRate.js index fd01fd014..6a871393d 100644 --- a/packages/caliper-core/lib/rate-control/replayRate.js +++ b/packages/caliper-core/lib/rate-control/replayRate.js @@ -117,7 +117,7 @@ class ReplayRateController extends RateInterface{ * @param {number} msg.txDuration The length of the round in SECONDS. * @param {number} msg.totalClients The number of clients executing the round. * @param {number} msg.clients The number of clients executing the round. - * @param {object} msg.clientargs Arguments for the client. + * @param {object} msg.clientArgs Arguments for the client. * @param {number} msg.clientIdx The 0-based index of the current client. * @param {number} msg.roundIdx The 1-based index of the current round. * @async diff --git a/packages/caliper-core/lib/report/report-builder.js b/packages/caliper-core/lib/report/report-builder.js index 1092b2db2..6347e62e2 100644 --- a/packages/caliper-core/lib/report/report-builder.js +++ b/packages/caliper-core/lib/report/report-builder.js @@ -15,7 +15,7 @@ 'use strict'; -const logger = require('../utils/caliper-utils').getLogger('caliper-flow'); +const Logger = require('../utils/caliper-utils').getLogger('caliper-flow'); const fs = require('fs'); const Mustache = require('mustache'); const path = require('path'); @@ -152,7 +152,7 @@ class ReportBuilder { this.data.tests[index].rounds.push({ 'id' : 'round ' + id, 'performance' : {'head':[], 'result': []}, - 'resource' : {'head':[], 'results': []} + 'resources': [] }); return id; } else { @@ -163,7 +163,7 @@ class ReportBuilder { 'rounds': [{ 'id' : 'round 0', 'performance' : {'head':[], 'result': []}, - 'resource' : {'head':[], 'results': []} + 'resources': [] }] }); return 0; @@ -207,7 +207,16 @@ class ReportBuilder { } /** - * set resource consumption table of a specific round + * Add new resource consumption table of a specific round within: + * { + * 'description' : this.descriptionmap.get(label), + * 'label' : label, + * 'rounds': [{ + * 'id' : 'round 0', + * 'performance' : {'head':[], 'result': []}, + * 'resources': [ {'head':[], 'results': []} ] + * }] + * } * @param {String} label the round label * @param {Number} id id of the round * @param {Array} table table array containing the resource consumption values @@ -234,13 +243,22 @@ class ReportBuilder { throw new Error('unrecognized report table'); } - this.data.tests[index].rounds[id].resource.head = table[0]; + const results = []; for(let i = 1 ; i < table.length ; i++) { if(!Array.isArray(table)) { throw new Error('unrecognized report table'); } - this.data.tests[index].rounds[id].resource.results.push({'result' : table[i]}); + results.push({'result' : table[i]}); } + + // Build a new object and add into resources array + const resource = { + head: table[0], + results + }; + this.data.tests[index].rounds[id].resources.push(resource); + + Logger.debug('resources count:', this.data.tests[index].rounds[id].resources.length); } /** @@ -275,9 +293,9 @@ class ReportBuilder { let html = Mustache.render(templateStr, this.data); try { await fs.writeFileSync(output, html); - logger.info(`Generated report with path ${output}`); + Logger.info(`Generated report with path ${output}`); } catch (err) { - logger.info(`Failed to generate report, with error ${err}`); + Logger.info(`Failed to generate report, with error ${err}`); throw err; } } diff --git a/packages/caliper-core/lib/report/report.js b/packages/caliper-core/lib/report/report.js index 99844469a..98cd7934e 100644 --- a/packages/caliper-core/lib/report/report.js +++ b/packages/caliper-core/lib/report/report.js @@ -15,9 +15,10 @@ 'use strict'; const ReportBuilder = require('./report-builder'); +const PrometheusQueryHelper = require('../prometheus/prometheus-query-helper'); const Blockchain = require('../blockchain'); const CaliperUtils = require('../utils/caliper-utils'); -const logger = CaliperUtils.getLogger('report-builder'); +const Logger = CaliperUtils.getLogger('report-builder'); const table = require('table'); @@ -27,13 +28,14 @@ const table = require('table'); class Report { /** - * Constructor - * @param {Monoitor} monitor the test monitor + * Constructor for the Report object + * @param {MonitorOrchestrator} monitorOrchestrator the test monitor */ - constructor(monitor) { - this.monitor = monitor; + constructor(monitorOrchestrator) { + this.monitorOrchestrator = monitorOrchestrator; this.reportBuilder = new ReportBuilder(); - this.resultsbyround = []; + this.resultsByRound = []; + this.queryClient = (monitorOrchestrator && monitorOrchestrator.hasMonitor('prometheus')) ? monitorOrchestrator.getMonitor('prometheus').getQueryClient() : null; } /** @@ -82,118 +84,264 @@ class Report { this.reportBuilder.addLabelDescriptionMap(config.test.rounds); } + /** + * Convert a result map to a table Array + * @param {Map | Map[]} resultMap a key/value result map or an array of such maps + * @return {sting[]} the table array + */ + convertToTable(resultMap) { + + // Format the Map into a table that may be printed using the npm table module + // tableRow[0] = array of column titles + // tableRow[1+] = array column values + + let tableArray = []; + if (Array.isArray(resultMap)){ + // More complex case, we have multiple results to deal with + for (const result of resultMap) { + const titles = []; + const values = []; + for (const key of result.keys()){ + titles.push(key); + values.push(result.get(key)); + } + if (!tableArray.length) { + tableArray.push(titles); + } + tableArray.push(values); + } + } else { + const titles = []; + const values = []; + for (const key of resultMap.keys()){ + titles.push(key); + values.push(resultMap.get(key)); + } + tableArray.push(titles); + tableArray.push(values); + } + return tableArray; + } + /** * print table - * @param {Array} value rows of the table + * @param {Map | Map[]} tableArray a table array containing performance information compatible with the npm table module */ - printTable(value) { - let t = table.table(value, {border: table.getBorderCharacters('ramac')}); - logger.info('\n' + t); + printTable(tableArray) { + // tableArray[0] = array of column titles + // tableArray[1+] = array column values + let t = table.table(tableArray, {border: table.getBorderCharacters('ramac')}); + Logger.info('\n' + t); } /** - * get the default result table's title - * @return {Array} row of the title + * Get a Map of result items + * @return {Map} Map of items to build results for, with default null entries */ - getResultTitle() { - // temporarily remove percentile return ['Name', 'Succ', 'Fail', 'Send Rate', 'Max Latency', 'Min Latency', 'Avg Latency', '75%ile Latency', 'Throughput']; - return ['Name', 'Succ', 'Fail', 'Send Rate', 'Max Latency', 'Min Latency', 'Avg Latency', 'Throughput']; + getResultColumnMap() { + const columns = ['Name', 'Succ', 'Fail', 'Send Rate (TPS)', 'Max Latency (s)', 'Min Latency (s)', 'Avg Latency (s)', 'Throughput (TPS)']; + const resultMap = new Map(); + + for (const item of columns) { + resultMap.set(item, 'N/A'); + } + + return resultMap; } /** - * get rows of the default result table - * @param {Array} r array of txStatistics JSON objects - * @return {Array} rows of the default result table + * Create a result map from locally gathered values + * @param {string} testLabel the test label name + * @param {JSON} results txStatistics JSON object + * @return {Map} a Map of key value pairing to create the default result table */ - getResultValue(r) { - let row = []; - try { - row.push(r.label); - row.push(r.succ); - row.push(r.fail); - (r.create.max === r.create.min) ? row.push((r.succ + r.fail) + ' tps') : row.push(((r.succ + r.fail) / (r.create.max - r.create.min)).toFixed(1) + ' tps'); - row.push(r.delay.max.toFixed(2) + ' s'); - row.push(r.delay.min.toFixed(2) + ' s'); - row.push((r.delay.sum / r.succ).toFixed(2) + ' s'); - - (r.final.last === r.create.min) ? row.push(r.succ + ' tps') : row.push((r.succ / (r.final.last - r.create.min)).toFixed(1) + ' tps'); - logger.debug('r.create.max: '+ r.create.max + ' r.create.min: ' + r.create.min + ' r.final.max: ' + r.final.max + ' r.final.min: '+ r.final.min + ' r.final.last: ' + r.final.last); - logger.debug(' throughput for only success time computed: '+ (r.succ / (r.final.max - r.create.min)).toFixed(1)); + getLocalResultValues(testLabel, results) { + Logger.debug ('getLocalResultValues called with: ', JSON.stringify(results)); + const resultMap = this.getResultColumnMap(); + resultMap.set('Name', testLabel ? testLabel : 'unknown'); + resultMap.set('Succ', results.hasOwnProperty('succ') ? results.succ : '-'); + resultMap.set('Fail', results.hasOwnProperty('fail') ? results.fail : '-'); + resultMap.set('Max Latency (s)', (results.hasOwnProperty('delay') && results.delay.hasOwnProperty('max')) ? results.delay.max.toFixed(2) : '-'); + resultMap.set('Min Latency (s)', (results.hasOwnProperty('delay') && results.delay.hasOwnProperty('min')) ? results.delay.min.toFixed(2) : '-'); + resultMap.set('Avg Latency (s)', (results.hasOwnProperty('delay') && results.delay.hasOwnProperty('sum') && results.hasOwnProperty('succ')) ? (results.delay.sum / results.succ).toFixed(2) : '-'); + + // Send rate needs a little more conditioning than sensible for a ternary op + if (results.hasOwnProperty('succ') && results.hasOwnProperty('fail') && results.hasOwnProperty('create') && results.create.hasOwnProperty('max') && results.create.hasOwnProperty('min')) { + const sendRate = (results.create.max === results.create.min) ? (results.succ + results.fail) : ((results.succ + results.fail) / (results.create.max - results.create.min)).toFixed(1); + resultMap.set('Send Rate (TPS)', sendRate); + } else { + resultMap.set('Send Rate (TPS)', '-'); } - catch (err) { - row = [r.label, 0, 0, 'N/A', 'N/A', 'N/A', 'N/A', 'N/A']; + + // Observed TPS needs a little more conditioning than sensible for a ternary op + if (results.hasOwnProperty('succ') && results.hasOwnProperty('final') && results.final.hasOwnProperty('last') && results.hasOwnProperty('create') && results.create.hasOwnProperty('min')) { + const tps = (results.final.last === results.create.min) ? results.succ : (results.succ / (results.final.last - results.create.min)).toFixed(1); + resultMap.set('Throughput (TPS)', tps); + } else { + resultMap.set('Throughput (TPS)', '-'); } - return row; + + return resultMap; } /** - * print the performance testing results of all test rounds + * Create a result mapping through querying a Prometheus server + * @param {string} testLabel the test label name + * @param {string} round the test round + * @param {number} startTime start time for Prometheus data query, in milliseconds since epoch + * @param {number} endTime end time for Prometheus data query, in milliseconds since epoch + * @return {Map} a Map of key value pairing to create the result table + */ + async getPrometheusResultValues(testLabel, round, startTime, endTime) { + const startQuery = startTime/1000; //convert to seconds + const endQuery = endTime/1000; + + const resultMap = this.getResultColumnMap(); + resultMap.set('Name', testLabel ? testLabel : 'unknown'); + + // Successful transactions + const txSuccessQuery = `sum(caliper_txn_success{instance="${testLabel}", round="${round}"})`; + const txSuccessCountResponse = await this.queryClient.query(txSuccessQuery, endQuery); + const txSuccessCount = txSuccessCountResponse ? PrometheusQueryHelper.extractFirstValueFromQueryResponse(txSuccessCountResponse) : '-'; + resultMap.set('Succ', txSuccessCount); + + // Failed transactions + const txFailQuery = `sum(caliper_txn_failure{instance="${testLabel}", round="${round}"})`; + const txFailCountResponse = await this.queryClient.query(txFailQuery, endQuery); + const txFailCount = txFailCountResponse ? PrometheusQueryHelper.extractFirstValueFromQueryResponse(txFailCountResponse) : '-'; + resultMap.set('Fail', txFailCount); + + // Maximum latency + const maxLatencyQuery = `max(caliper_latency{instance="${testLabel}", round="${round}"})`; + const maxLatenciesResponse = await this.queryClient.rangeQuery(maxLatencyQuery, startQuery, endQuery); + const maxLatenciesStats = PrometheusQueryHelper.extractStatisticFromRange(maxLatenciesResponse, 'max'); + const maxLatency = maxLatenciesStats.has('unknown') ? maxLatenciesStats.get('unknown').toFixed(2) : '-'; + resultMap.set('Max Latency (s)', maxLatency); + + // Min latency + const minLatencyQuery = `min(caliper_latency{instance="${testLabel}", round="${round}"})`; + const minLatenciesResponse = await this.queryClient.rangeQuery(minLatencyQuery, startQuery, endQuery); + const minLatenciesStats = PrometheusQueryHelper.extractStatisticFromRange(minLatenciesResponse, 'min'); + const minLatency = minLatenciesStats.has('unknown') ? minLatenciesStats.get('unknown').toFixed(2) : '-'; + resultMap.set('Min Latency (s)', minLatency); + + // Avg Latency + const avgLatencyQuery = `avg(caliper_latency{instance="${testLabel}", round="${round}"})`; + const avgLatenciesResponse = await this.queryClient.rangeQuery(avgLatencyQuery, startQuery, endQuery); + const avgLatenciesStats = PrometheusQueryHelper.extractStatisticFromRange(avgLatenciesResponse, 'avg'); + const avgLatency = avgLatenciesStats.has('unknown') ? avgLatenciesStats.get('unknown').toFixed(2) : '-'; + resultMap.set('Avg Latency (s)', avgLatency); + + // Avg Submit Rate within the time bounding + const avgSubmitRateQuery = `sum(caliper_txn_submit_rate{instance="${testLabel}", round="${round}"})`; + const avgSubmitRateResponse = await this.queryClient.rangeQuery(avgSubmitRateQuery, startQuery, endQuery); + const avgSubmitRateStats = PrometheusQueryHelper.extractStatisticFromRange(avgSubmitRateResponse, 'avg'); + const avgSubmitRate = avgSubmitRateStats.has('unknown') ? avgSubmitRateStats.get('unknown').toFixed(2) : '-'; + resultMap.set('Send Rate (TPS)', avgSubmitRate); + + // Average TPS (completed transactions) + const tps = ((txSuccessCount + txFailCount)/(endQuery - startQuery)).toFixed(1); + resultMap.set('Throughput (TPS)', tps); + + return resultMap; + } + + /** + * Print the performance testing results of all test rounds */ printResultsByRound() { - this.resultsbyround[0].unshift('Test'); - for(let i = 1 ; i < this.resultsbyround.length ; i++) { - this.resultsbyround[i].unshift(i.toFixed(0)); - } - logger.info('###all test results:###'); - this.printTable(this.resultsbyround); + const tableArray = this.convertToTable(this.resultsByRound); + Logger.info('### All test results ###'); + this.printTable(tableArray); - this.reportBuilder.setSummaryTable(this.resultsbyround); + this.reportBuilder.setSummaryTable(tableArray); } /** - * merge testing results from various clients and store the merged result in the global result array - * txStatistics = { - * succ : , // number of committed txns - * fail : , // number of failed txns - * create : {min:, max: }, // min/max time when txns were created/submitted - * final : {min:, max: }, // min/max time when txns were committed - * delay : {min:, max: , sum:, detail:[]}, // min/max/sum of txns' end2end delay, as well as all txns' delay - * } - * @param {Array} results array of txStatistics + * Augment the report with details generated through extraction from local process reporting + * @param {JSON} results JSON object {tsStats: txStats[], start: number, end: number} containing an array of txStatistics * @param {String} label label of the test round - * @return {Promise} promise object + * @return {Promise} promise object containing the current report index */ - processResult(results, label){ - try{ - let resultTable = []; - resultTable[0] = this.getResultTitle(); - let r; - if(Blockchain.mergeDefaultTxStats(results) === 0) { - r = Blockchain.createNullDefaultTxStats(); - r.label = label; - } - else { - r = results[0]; - r.label = label; - resultTable[1] = this.getResultValue(r); - } + async processLocalTPSResults(results, label){ + try { + let resultSet; - let sTP = r.sTPTotal / r.length; - let sT = r.sTTotal / r.length; - logger.debug('sendTransactionProposal: ' + sTP + 'ms length: ' + r.length); - logger.debug('sendTransaction: ' + sT + 'ms'); - logger.debug('invokeLatency: ' + r.invokeTotal / r.length + 'ms'); - if(this.resultsbyround.length === 0) { - this.resultsbyround.push(resultTable[0].slice(0)); + if (Blockchain.mergeDefaultTxStats(results) === 0) { + resultSet = Blockchain.createNullDefaultTxStats(); + } else { + resultSet = results[0]; } - if(resultTable.length > 1) { - this.resultsbyround.push(resultTable[1].slice(0)); - } - logger.info('###test result:###'); - this.printTable(resultTable); + + const resultMap = this.getLocalResultValues(label, resultSet); + // Add this set of results to the main round collection + this.resultsByRound.push(resultMap); + + // Print TPS result for round to console + Logger.info('### Test result ###'); + const tableArray = this.convertToTable(resultMap); + this.printTable(tableArray); + + // Add TPS to the report let idx = this.reportBuilder.addBenchmarkRound(label); - this.reportBuilder.setRoundPerformance(label, idx, resultTable); - let resourceTable = this.monitor.getDefaultStats(); - if(resourceTable.length > 0) { - logger.info('### resource stats ###'); + this.reportBuilder.setRoundPerformance(label, idx, tableArray); + + return idx; + } catch(error) { + Logger.error(`processLocalTPSResults failed with error: ${error}`); + throw error; + } + } + + /** + * Retrieve the resource monitor statistics and add to the report index + * @param {number} idx the report index to add the resource statistics under + * @param {string} label the test label + */ + async buildRoundResourceStatistics(idx, label) { + // Retrieve statistics from all monitors + const types = this.monitorOrchestrator.getAllMonitorTypes(); + for (let type of types) { + const statsMap = await this.monitorOrchestrator.getStatisticsForMonitor(type); + const resourceTable = this.convertToTable(statsMap); + if (resourceTable.length > 0) { + // print to console for view and add to report + Logger.info(`### ${type} resource stats ###'`); this.printTable(resourceTable); this.reportBuilder.setRoundResource(label, idx, resourceTable); } - return Promise.resolve(); } - catch(err) { - logger.error(err); - return Promise.reject(err); + } + + /** + * Augment the report with details generated through extraction from Prometheus + * @param {JSON} timing JSON object containing start/end times required to query + * the prometheus server + * @param {String} label label of the test round + * @param {number} round the current test round + * @return {Promise} promise object containing the report index + * @async + */ + async processPrometheusTPSResults(timing, label, round){ + try { + const resultMap = await this.getPrometheusResultValues(label, round, timing.start, timing.end); + + // Add this set of results to the main round collection + this.resultsByRound.push(resultMap); + + // Print TPS result for round to console + Logger.info('### Test result ###'); + const tableArray = this.convertToTable(resultMap); + this.printTable(tableArray); + + // Add TPS to the report + let idx = this.reportBuilder.addBenchmarkRound(label); + this.reportBuilder.setRoundPerformance(label, idx, tableArray); + + return idx; + } catch (error) { + Logger.error(`processPrometheusTPSResults failed with error: ${error}`); + throw error; } } diff --git a/packages/caliper-core/lib/report/template/report.html b/packages/caliper-core/lib/report/template/report.html index 3cf1c4e7c..93746bc07 100644 --- a/packages/caliper-core/lib/report/template/report.html +++ b/packages/caliper-core/lib/report/template/report.html @@ -139,7 +139,7 @@

{{label}}

{{description}}

{{#rounds}}

{{id}}

- performance metrics + Performance metrics {{#performance}} @@ -150,19 +150,23 @@

{{id}}

{{/performance}} - resource consumption - {{#resource}} + Resource consumption + {{#resources}} - {{#head}} {{/head}} - - {{#results}} - - {{#result}} {{/result}} +
{{.}}
{{.}}
+ + {{#head}} {{/head}} + + {{#results}} + + {{#result}} {{/result}} + + {{/results}} +
{{.}}
{{.}}
- {{/results}} - {{/resource}} + {{/resources}} {{/rounds}} {{/tests}} diff --git a/packages/caliper-core/lib/test-observers/local-observer.js b/packages/caliper-core/lib/test-observers/local-observer.js new file mode 100644 index 000000000..c778c73d7 --- /dev/null +++ b/packages/caliper-core/lib/test-observers/local-observer.js @@ -0,0 +1,265 @@ +/* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + + +'use strict'; + +const TestObserverInterface = require('./observer-interface'); +const Utils = require('../utils/caliper-utils'); +const Logger = Utils.getLogger('local-observer'); + +/** + * LocalObserver class used to observe test statistics via terminal + */ +class LocalObserver extends TestObserverInterface { + + /** + * Constructor + * @param {String} configPath path of the configuration file + */ + constructor(configPath) { + super(configPath); + + // set the observer interval + const interval = (this.config.observer && this.config.observer.interval) ? this.config.observer.interval : 1; + this.observeInterval = interval * 1000; + Logger.info(`Observer interval set to ${interval} seconds`); + this.observeIntervalObject = null; + this.updateTail = 0; + this.updateID = 0; + this.testData = { + maxlen : 300, + throughput: { + x: [], + submitted: [0], + succeeded: [0], + failed: [0] + }, + latency: { + x: [], + max: [0], + min: [0], + avg: [0] + }, + summary: { + txSub: 0, + txSucc: 0, + txFail: 0, + round: 0, + }, + report: '' + }; + } + + /** + * Add Throughput + * @param {*} sub submitted + * @param {*} suc successful + * @param {*} fail fail + */ + addThroughput(sub, suc, fail) { + this.testData.throughput.submitted.push(sub/this.observeInterval); + this.testData.throughput.succeeded.push(suc/this.observeInterval); + this.testData.throughput.failed.push(fail/this.observeInterval); + if (this.testData.throughput.x.length < this.testData.throughput.submitted.length) { + let last = this.testData.throughput.x[this.testData.throughput.x.length - 1]; + this.testData.throughput.x.push(last + this.observeInterval); + } + if (this.testData.throughput.submitted.length > this.testData.maxlen) { + this.testData.throughput.submitted.shift(); + this.testData.throughput.succeeded.shift(); + this.testData.throughput.failed.shift(); + this.testData.throughput.x.shift(); + } + this.testData.summary.txSub += sub; + this.testData.summary.txSucc += suc; + this.testData.summary.txFail += fail; + } + + /** + * Add Latency + * @param {*} max the maximum + * @param {*} min the minimum + * @param {*} avg the average + */ + addLatency(max, min, avg) { + this.testData.latency.max.push(max); + this.testData.latency.min.push(min); + this.testData.latency.avg.push(avg); + if(this.testData.latency.x.length < this.testData.latency.max.length) { + let last = this.testData.latency.x[this.testData.latency.x.length - 1]; + this.testData.latency.x.push(last + this.observeInterval); + } + if (this.testData.latency.max.length > this.testData.maxlen) { + this.testData.latency.max.shift(); + this.testData.latency.min.shift(); + this.testData.latency.avg.shift(); + this.testData.latency.x.shift(); + } + } + + /** + * Reset test data + */ + resetTestData() { + this.testData.throughput = { + x: [], + submitted: [0], + succeeded: [0], + failed: [0] + }; + this.testData.latency = { + x: [], + max: [0], + min: [0], + avg: [0] + }; + this.testData.summary = { + txSub: 0, + txSucc: 0, + txFail: 0, + round: this.testData.summary.round, + }; + } + + /** + * Refresh data + * @param {*} updates updates to use + */ + refreshData(updates) { + if (updates.length === 0 || Object.entries(updates[0]).length === 0) { + this.addThroughput(0,0,0); + this.addLatency(0,0,0); + } else { + let sub = 0, suc = 0, fail = 0; + let deMax = -1, deMin = -1, deAvg = 0; + for(let i = 0 ; i < updates.length ; i++) { + let data = updates[i]; + if (data.type.localeCompare('txReset') === 0) { + // Resetting values + Logger.info('Resetting txCount indicator count'); + this.resetTestData(); + continue; + } + sub += data.submitted; + suc += data.committed.succ; + fail += data.committed.fail; + + if(data.committed.succ > 0) { + if(deMax === -1 || deMax < data.committed.delay.max) { + deMax = data.committed.delay.max; + } + if(deMin === -1 || deMin > data.committed.delay.min) { + deMin = data.committed.delay.min; + } + deAvg += data.committed.delay.sum; + } + } + if(suc > 0) { + deAvg /= suc; + } + this.addThroughput(sub, suc, fail); + + if(isNaN(deMax) || isNaN(deMin) || deAvg === 0) { + this.addLatency(0,0,0); + } + else { + this.addLatency(deMax, deMin, deAvg); + } + + } + + Logger.info('[' + this.testName + ' Round ' + this.testRound + ' Transaction Info] - Submitted: ' + this.testData.summary.txSub + + ' Succ: ' + this.testData.summary.txSucc + + ' Fail:' + this.testData.summary.txFail + + ' Unfinished:' + (this.testData.summary.txSub - this.testData.summary.txSucc - this.testData.summary.txFail)); + } + + /** + * Perform an update + * @async + */ + async update() { + if (typeof this.clientOrchestrator === 'undefined') { + this.refreshData([]); + return; + } + let updates = this.clientOrchestrator.getUpdates(); + if(updates.id > this.updateID) { // new buffer + this.updateTail = 0; + this.updateID = updates.id; + } + let data = []; + let len = updates.data.length; + if(len > this.updateTail) { + data = updates.data.slice(this.updateTail, len); + this.updateTail = len; + } + this.refreshData(data); + } + + /** + * Start watching the test output of the orchestrator + * @param {ClientOrchestrator} clientOrchestrator the client orchestrator + */ + startWatch(clientOrchestrator) { + this.clientOrchestrator = clientOrchestrator; + if(this.observeIntervalObject === null) { + this.updateTail = 0; + this.updateID = 0; + // start an interval to query updates + const self = this; + this.observeIntervalObject = setInterval(async() => { await self.update(); }, this.observeInterval); + } + } + + /** + * Stop watching the test output of the orchestrator + * @async + */ + async stopWatch() { + if(this.observeIntervalObject) { + clearInterval(this.observeIntervalObject); + this.observeIntervalObject = null; + } + await Utils.sleep(this.observeInterval); + this.update(); + } + + /** + * Set the test name to be reported + * @param {String} name the benchmark name + */ + setBenchmark(name) { + this.testName = name; + } + /** + * Set the test round for the watcher + * @param{*} roundIdx the round index + */ + setRound(roundIdx) { + this.testRound = roundIdx; + } +} + +/** + * Creates a new rate controller instance. + * @param {String} absConfigFile The absolute path to the benchmark config file + * @return {ObserverInterface} The rate controller instance. + */ +function createTestObserver(absConfigFile) { + return new LocalObserver(absConfigFile); +} + +module.exports.createTestObserver = createTestObserver; diff --git a/packages/caliper-core/lib/test-observers/null-observer.js b/packages/caliper-core/lib/test-observers/null-observer.js new file mode 100644 index 000000000..3e03a0cd9 --- /dev/null +++ b/packages/caliper-core/lib/test-observers/null-observer.js @@ -0,0 +1,86 @@ +/* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + + +'use strict'; + +const TestObserverInterface = require('./observer-interface'); +const Utils = require('../utils/caliper-utils'); +const Logger = Utils.getLogger('null-observer'); + +/** + * NullObserver class used to omit test statistics observation + */ +class NullObserver extends TestObserverInterface { + + /** + * Constructor + * @param {String} configPath path of the configuration file + */ + constructor(configPath) { + super(configPath); + Logger.info('Configured observer'); + } + + /** + * Perform an update + */ + async update() { + // No action + Logger.debug('No action taken by NullObserver on update'); + } + + /** + * Start observing the test output + * @param {ClientOrchestrator} clientOrchestrator the client orchestrator + */ + startWatch(clientOrchestrator) { + Logger.debug('No action taken by NullObserver on startWatch'); + } + + /** + * Stop watching the test output + * @async + */ + async stopWatch() { + Logger.debug('No action taken by NullObserver on stopWatch'); + } + + /** + * Set the test name to be reported + * @param {String} name the benchmark name + */ + setBenchmark(name) { + Logger.debug('No action taken by NullObserver on setBenchmark'); + } + /** + * Set the test round for the watcher + * @param{*} roundIdx the round index + */ + setRound(roundIdx) { + Logger.debug('No action taken by NullObserver on setRound'); + } + +} + +/** + * Creates a new rate controller instance. + * @param {String} absConfigFile The absolute path to the benchmark config file + * @return {ObserverInterface} The rate controller instance. + */ +function createTestObserver(absConfigFile) { + return new NullObserver(absConfigFile); +} + +module.exports.createTestObserver = createTestObserver; diff --git a/packages/caliper-core/lib/test-observers/observer-interface.js b/packages/caliper-core/lib/test-observers/observer-interface.js new file mode 100644 index 000000000..f66d90600 --- /dev/null +++ b/packages/caliper-core/lib/test-observers/observer-interface.js @@ -0,0 +1,75 @@ +/* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +'use strict'; + +const Util = require('../utils/caliper-utils'); + +/** + * Interface of test observer. Test observer implementations must follow a naming convention that is -observer.js so + * that they may be dynamically loaded within Caliper flow + */ +class TestObserverInterface { + + /** + * Constructor + * @param {string} configPath the config file path + */ + constructor(configPath) { + this.config = Util.parseYaml(configPath); + } + + /** + * Perform an update + * @async + */ + async update() { + throw new Error('update is not implemented for this test observer'); + } + + /** + * Start watching the test output from the orchestrator + * @param {ClientOrchestrator} clientOrchestrator the client orchestrator + */ + startWatch(clientOrchestrator) { + throw new Error('startWatch is not implemented for this test observer'); + } + + /** + * Stop watching the test output from the orchestrator + * @async + */ + async stopWatch() { + throw new Error('stopWatch is not implemented for this test observer'); + } + + /** + * Set the test name to be reported + * @param {String} name the benchmark name + */ + setBenchmark(name) { + throw new Error('setBenchmark is not implemented for this test observer'); + } + + /** + * Set the test round for the observer + * @param{*} roundIdx the round index + */ + setRound(roundIdx) { + throw new Error('setRound is not implemented for this test observer'); + } + +} + +module.exports = TestObserverInterface; diff --git a/packages/caliper-core/lib/test-observers/prometheus-observer.js b/packages/caliper-core/lib/test-observers/prometheus-observer.js new file mode 100644 index 000000000..33f12f091 --- /dev/null +++ b/packages/caliper-core/lib/test-observers/prometheus-observer.js @@ -0,0 +1,126 @@ +/* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + + +'use strict'; + +const TestObserverInterface = require('./observer-interface'); +const PrometheusQueryClient = require('../prometheus/prometheus-query-client'); +const PrometheusQueryHelper = require('../prometheus/prometheus-query-helper'); +const Utils = require('../utils/caliper-utils'); +const Logger = Utils.getLogger('prometheus-observer'); + +/** + * PrometheusObserver class used to observe test statistics via terminal + */ +class PrometheusObserver extends TestObserverInterface { + + /** + * Constructor + * @param {String} configPath path of the configuration file + */ + constructor(configPath) { + super(configPath); + + // determine interval + const interval = (this.config.observer && this.config.observer.interval) ? this.config.observer.interval : 1; + this.observeInterval = interval * 1000; + + // Define the query client + const queryUrl = this.config.monitor.prometheus.url; + this.queryClient = new PrometheusQueryClient(queryUrl); + Logger.info(`Configured observer to query URL ${queryUrl} every ${interval} seconds`); + } + + /** + * Perform an update + */ + async update() { + // Update using Prometheus Query + // -Successful transactions + const txSuccessQuery = `sum(caliper_txn_success{instance="${this.testName}", round="${this.testRound}"})`; + const txSuccessCountResponse = await this.queryClient.query(txSuccessQuery, Date.now()/1000); + const txSuccessCount = txSuccessCountResponse ? PrometheusQueryHelper.extractFirstValueFromQueryResponse(txSuccessCountResponse) : '-'; + + // -Failed transactions + const txFailQuery = `sum(caliper_txn_failure{instance="${this.testName}", round="${this.testRound}"})`; + const txFailCountResponse = await this.queryClient.query(txFailQuery, Date.now()/1000); + const txFailCount = txFailCountResponse ? PrometheusQueryHelper.extractFirstValueFromQueryResponse(txFailCountResponse) : '-'; + + // -Pending transactions + const txPendingQuery = `sum(caliper_txn_pending{instance="${this.testName}", round="${this.testRound}"})`; + const txPendingCountResponse = await this.queryClient.query(txPendingQuery, Date.now()/1000); + const txPendingCount = txPendingCountResponse ? PrometheusQueryHelper.extractFirstValueFromQueryResponse(txPendingCountResponse) : '-'; + + // Could use query on submitted, but quicker to do addition here than use query service + Logger.info('[' + this.testName + ' Round ' + this.testRound + ' Transaction Info] - Submitted: ' + (txSuccessCount + txFailCount + txPendingCount) + + ' Succ: ' + txSuccessCount + + ' Fail:' + txFailCount + + ' Unfinished:' + txPendingCount); + } + + /** + * Start observing the test output + * @param {ClientOrchestrator} clientOrchestrator the client orchestrator + */ + startWatch(clientOrchestrator) { + Logger.info(`Starting observer cycle with interval ${this.observeInterval} ms`); + this.clientOrchestrator = clientOrchestrator; + if(!this.observeIntervalObject) { + // start an interval to query updates + const self = this; + this.observeIntervalObject = setInterval(async() => { await self.update(); }, this.observeInterval); + } + } + + /** + * Stop watching the test output + * @async + */ + async stopWatch() { + if(this.observeIntervalObject) { + clearInterval(this.observeIntervalObject); + this.observeIntervalObject = null; + } + await Utils.sleep(this.observeInterval); + this.update(); + } + + /** + * Set the test name to be reported + * @param {String} name the benchmark name + */ + setBenchmark(name) { + this.testName = name; + } + /** + * Set the test round for the watcher + * @param{*} roundIdx the round index + */ + setRound(roundIdx) { + this.testRound = roundIdx; + } + +} + +/** + * Creates a new rate controller instance. + * @param {String} absConfigFile The absolute path to the benchmark config file + * @return {ObserverInterface} The rate controller instance. + */ +function createTestObserver(absConfigFile) { + return new PrometheusObserver(absConfigFile); +} + +module.exports.createTestObserver = createTestObserver; diff --git a/packages/caliper-core/lib/test-observers/test-observer.js b/packages/caliper-core/lib/test-observers/test-observer.js new file mode 100644 index 000000000..0903c96cf --- /dev/null +++ b/packages/caliper-core/lib/test-observers/test-observer.js @@ -0,0 +1,89 @@ +/* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +'use strict'; + +const CaliperUtils = require('../utils/caliper-utils'); +const Logger = CaliperUtils.getLogger('testObserver.js'); + +const builtInObservers = new Map([ + ['none', './null-observer'], + ['local', './local-observer.js'], + ['prometheus', './prometheus-observer.js'] +]); + +const TestObserver = class { + + /** + * Instantiates the proxy test observer and creates the configured observer behind it. + * @param {String} observerType The observer to use. + * @param {String} absConfigFile The absolute path to the benchmark config file + */ + constructor(observerType, absConfigFile) { + Logger.debug(`Creating test observer of type ${observerType}`); + + // resolve the type to a module path + let modulePath = builtInObservers.has(observerType) + ? builtInObservers.get(observerType) : CaliperUtils.resolvePath(observerType); + + let factoryFunction = require(modulePath).createTestObserver; + if (!factoryFunction) { + throw new Error(`${observerType} does not export the mandatory factory function 'createTestObserver'`); + } + + this.observer = factoryFunction(absConfigFile); + } + + /** + * Perform an update + * @async + */ + async update() { + await this.observer.update(); + } + + /** + * Start watching the test output from the orchestrator + * @param {ClientOrchestrator} clientOrchestrator the client orchestrator + */ + startWatch(clientOrchestrator) { + this.observer.startWatch(clientOrchestrator); + } + + /** + * Stop watching the test output from the orchestrator + * @async + */ + async stopWatch() { + await this.observer.stopWatch(); + } + + /** + * Set the test name to be reported + * @param {String} name the benchmark name + */ + setBenchmark(name) { + this.observer.setBenchmark(name); + } + + /** + * Set the test round for the observer + * @param{*} roundIdx the round index + */ + setRound(roundIdx) { + this.observer.setRound(roundIdx); + } + +}; + +module.exports = TestObserver; diff --git a/packages/caliper-core/lib/test/defaultTest.js b/packages/caliper-core/lib/test-runners/default-test.js similarity index 69% rename from packages/caliper-core/lib/test/defaultTest.js rename to packages/caliper-core/lib/test-runners/default-test.js index 5dbefd99b..31da79ee2 100644 --- a/packages/caliper-core/lib/test/defaultTest.js +++ b/packages/caliper-core/lib/test-runners/default-test.js @@ -33,10 +33,10 @@ class DefaultTest { * @param {Object} clientFactory factory used to spawn test clients * @param {String} networkRoot the root location * @param {Report} report the report being built - * @param {Object} demo the demo UI component - * @param {Object} monitor The monitor object + * @param {TestObserver} testObserver the test observer + * @param {Object} monitorOrchestrator The monitor object */ - constructor(clientArgs, absNetworkFile, clientOrchestrator, clientFactory, networkRoot, report, demo, monitor) { + constructor(clientArgs, absNetworkFile, clientOrchestrator, clientFactory, networkRoot, report, testObserver, monitorOrchestrator) { this.clientArgs = clientArgs; this.absNetworkFile = absNetworkFile; this.clientFactory = clientFactory; @@ -44,8 +44,8 @@ class DefaultTest { this.networkRoot = networkRoot; this.report = report; this.round = 0; - this.demo = demo; - this.monitor = monitor; + this.testObserver = testObserver; + this.monitorOrchestrator = monitorOrchestrator; } /** @@ -57,6 +57,7 @@ class DefaultTest { */ async runTestRounds(args, final) { logger.info(`####### Testing '${args.label}' #######`); + this.testObserver.setBenchmark(args.label); const testLabel = args.label; const testRounds = args.txDuration ? args.txDuration : args.txNumber; const tests = []; // array of all test rounds @@ -64,7 +65,7 @@ class DefaultTest { // Build test rounds for (let i = 0 ; i < testRounds.length ; i++) { - const msg = { + const test = { type: 'test', label : testLabel, rateControl: args.rateControl[i] ? args.rateControl[i] : {type:'fixed-rate', 'opts' : {'tps': 1}}, @@ -72,17 +73,19 @@ class DefaultTest { args: args.arguments, cb : args.callback, config: configPath, - root: this.networkRoot + root: this.networkRoot, + testRound: i, + pushUrl: this.monitorOrchestrator.hasMonitor('prometheus') ? this.monitorOrchestrator.getMonitor('prometheus').getPushGatewayURL() : null }; // condition for time based or number based test driving if (args.txNumber) { - msg.numb = testRounds[i]; + test.numb = testRounds[i]; } else if (args.txDuration) { - msg.txDuration = testRounds[i]; + test.txDuration = testRounds[i]; } else { throw new Error('Unspecified test driving mode'); } - tests.push(msg); + tests.push(test); } @@ -95,12 +98,24 @@ class DefaultTest { logger.info(`------ Test round ${this.round += 1} ------`); testIdx++; - test.roundIdx = this.round; // propagate round ID to clients - this.demo.startWatch(this.clientOrchestrator); + this.testObserver.setRound(test.testRound); try { - await this.clientOrchestrator.startTest(test, this.clientArgs, this.report, testLabel, this.clientFactory); + this.testObserver.startWatch(this.clientOrchestrator); + const {results, start, end} = await this.clientOrchestrator.startTest(test, this.clientArgs, this.clientFactory); + await this.testObserver.stopWatch(); + + // Build the report + // - TPS + let idx; + if (this.monitorOrchestrator.hasMonitor('prometheus')) { + idx = await this.report.processPrometheusTPSResults({start, end}, testLabel, test.testRound); + } else { + idx = await this.report.processLocalTPSResults(results, testLabel); + } + + // - Resource utilization + await this.report.buildRoundResourceStatistics(idx, testLabel); - this.demo.pauseWatch(); successes++; logger.info(`------ Passed '${testLabel}' testing ------`); @@ -108,10 +123,9 @@ class DefaultTest { if(!final || testIdx !== tests.length) { logger.info('Waiting 5 seconds for the next round...'); await CaliperUtils.sleep(5000); - await this.monitor.restart(); + await this.monitorOrchestrator.restartAllMonitors(); } } catch (err) { - this.demo.pauseWatch(); failures++; logger.error(`------ Failed '${testLabel}' testing with the following error ------ ${err.stack ? err.stack : err}`); diff --git a/packages/caliper-core/lib/utils/benchmark-validator.js b/packages/caliper-core/lib/utils/benchmark-validator.js new file mode 100644 index 000000000..a2dc0499f --- /dev/null +++ b/packages/caliper-core/lib/utils/benchmark-validator.js @@ -0,0 +1,94 @@ +/* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + + +'use strict'; + +const Logger = require('./caliper-utils').getLogger('benchmark-validator'); + +const PermittedObservers = ['none', 'local', 'prometheus']; +/** + * Class for Benchmark validation + */ +class BenchmarkValidator { + + /** + * Validate the bench configuration object + * @param {Object} benchConfig the bench configuration object + */ + static validateObject(benchConfig) { + BenchmarkValidator.validateObserver(benchConfig); + } + + /** + * Validate the observer specified in the bench configuration object + * @param {Object} benchConfig the bench configuration object + */ + static validateObserver(benchConfig) { + // Must be specified + if (!benchConfig.hasOwnProperty('observer')) { + Logger.info('No observer specified, will default to `none`'); + return; + } + // If specified must have type specified [local, prometheus] + if (!benchConfig.observer.hasOwnProperty('type')) { + BenchmarkValidator.throwMissingPropertyBenchmarkError('observer.type'); + } + if (!PermittedObservers.includes(benchConfig.observer.type)) { + BenchmarkValidator.throwInvalidPropertyBenchmarkError('observer.type', benchConfig.observer.type); + } + + // Must have an integer interval specified if a non-null observer specified + if (!(benchConfig.observer.type.localeCompare('none') === 0) && !benchConfig.observer.hasOwnProperty('interval')) { + BenchmarkValidator.throwMissingPropertyBenchmarkError('observer.interval'); + } + if (!PermittedObservers.includes(benchConfig.observer.type)) { + BenchmarkValidator.throwInvalidPropertyBenchmarkError('observer.type', benchConfig.observer.type); + } + + // If prometheus monitor specified, must be a prometheus observer + if (benchConfig.monitor.type.includes('prometheus') && !(benchConfig.observer.type.localeCompare('prometheus') === 0) ) { + BenchmarkValidator.throwIncompatibleTypeBenchmarkError('observer.type.local', 'monitor.type.prometheus'); + } + } + + /** + * Throw a consistent error + * @param {String} property property missing from benchmark configuration file + */ + static throwMissingPropertyBenchmarkError(property) { + throw new Error(`Benchmark configuration is missing required property '${property}'`); + } + + /** + * Throw a consistent error + * @param {String} property property missing from benchmark configuration file + * @param {String} value the set value + */ + static throwInvalidPropertyBenchmarkError(property, value) { + throw new Error(`Benchmark configuration has an invalid property '${property}' value of '${value}'`); + } + + /** + * Throw a consistent error + * @param {String} properties0 the set properties0 + * @param {String} properties1 the set properties1 + */ + static throwIncompatibleTypeBenchmarkError(properties0, properties1) { + throw new Error(`Benchmark configuration has an incompatible types of '${properties0}' and '${properties1}'`); + } + +} + +module.exports = BenchmarkValidator; diff --git a/packages/caliper-core/package.json b/packages/caliper-core/package.json index 52c194556..70e6a19a9 100644 --- a/packages/caliper-core/package.json +++ b/packages/caliper-core/package.json @@ -1,5 +1,6 @@ { "name": "@hyperledger/caliper-core", + "description": "Core Hyperledger Caliper module, used for running performance benchmarks that interact with blockchain technologies", "version": "0.1.0", "repository": { "type": "git", @@ -41,7 +42,7 @@ "mocha": "3.4.2", "nyc": "11.1.0", "rewire": "^4.0.0", - "sinon": "2.3.8", + "sinon": "^7.3.2", "license-check-and-add": "2.3.6" }, "license-check-and-add-config": { @@ -86,14 +87,14 @@ }, "nyc": { "exclude": [ - "lib/**" + "test/**" ], "reporter": [ "text-summary", "html" ], "all": true, - "check-coverage": false, + "check-coverage": true, "statements": 1, "branches": 1, "functions": 1, diff --git a/packages/caliper-core/test/client/client-orchestrator.js b/packages/caliper-core/test/client/client-orchestrator.js new file mode 100644 index 000000000..e69359489 --- /dev/null +++ b/packages/caliper-core/test/client/client-orchestrator.js @@ -0,0 +1,167 @@ +/* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +'use strict'; + +const rewire = require('rewire'); +const ClientOrchestratorRewire = rewire('../../lib/client/client-orchestrator'); + +const chai = require('chai'); +chai.should(); +const sinon = require('sinon'); + +describe('client orchestrator implementation', () => { + + + describe('#init', () => { + + it('should read the number of test clients if present in the config file', () => { + const FakeParseYaml = sinon.stub().returns({ test: { clients: {number: 7}}}); + ClientOrchestratorRewire.__set__('util.parseYaml', FakeParseYaml); + + const myOrchestrator = new ClientOrchestratorRewire(); + myOrchestrator.init(); + myOrchestrator.number.should.equal(7); + + }); + + it('should default to one client in the test if not specified in the config file ', () => { + const FakeParseYaml = sinon.stub().returns({ test: { clients: {notNumber: 2}}}); + ClientOrchestratorRewire.__set__('util.parseYaml', FakeParseYaml); + + const myOrchestrator = new ClientOrchestratorRewire(); + myOrchestrator.init(); + myOrchestrator.number.should.equal(1); + }); + }); + + describe('#startTest', () => { + + const FakeParseYaml = sinon.stub().returns({ test: { clients: {number: 7}}}); + ClientOrchestratorRewire.__set__('util.parseYaml', FakeParseYaml); + const myOrchestrator = new ClientOrchestratorRewire(); + myOrchestrator.init(); + let _startTestStub; + let formatResultsStub; + + beforeEach(() => { + _startTestStub = sinon.stub(); + formatResultsStub = sinon.stub().returns('formatted'); + myOrchestrator._startTest = _startTestStub; + myOrchestrator.formatResults = formatResultsStub; + }); + + it('should increment the updates.id variable', async () => { + myOrchestrator.updates.id = 41; + const testMsg = {msg: 'test msg'}; + const clientArgs = [1,2,3]; + const factory = {}; + + await myOrchestrator.startTest(testMsg, clientArgs, factory); + + myOrchestrator.updates.id.should.equal(42); + }); + + it('should call _startTest with known arguments', async () => { + myOrchestrator.updates.id = 0; + const testMsg = {msg: 'test msg'}; + const clientArgs = [1,2,3]; + const factory = {factory: 'a factory'}; + + await myOrchestrator.startTest(testMsg, clientArgs, factory); + + sinon.assert.calledOnce(_startTestStub); + sinon.assert.calledWith(_startTestStub, 7, testMsg, clientArgs, [], [], factory); + }); + + it('should call formatResults', async() => { + myOrchestrator.updates.id = 0; + const testMsg = {msg: 'test msg'}; + const clientArgs = [1,2,3]; + const factory = {}; + + await myOrchestrator.startTest(testMsg, clientArgs, factory); + + sinon.assert.calledOnce(formatResultsStub); + }); + + it('should return the response from formatResults', async () => { + myOrchestrator.updates.id = 0; + const testMsg = {msg: 'test msg'}; + const clientArgs = [1,2,3]; + const factory = {}; + + const response = await myOrchestrator.startTest(testMsg, clientArgs, factory); + response.should.equal('formatted'); + }); + + }); + + + describe('#getUpdates', () => { + + const FakeParseYaml = sinon.stub().returns({ test: { clients: {number: 7}}}); + ClientOrchestratorRewire.__set__('util.parseYaml', FakeParseYaml); + const myOrchestrator = new ClientOrchestratorRewire(); + + it('should return the updates', () => { + const checkVal = 'this is my update'; + // overwrite with known value + myOrchestrator.updates = checkVal; + // assert repsonse + myOrchestrator.getUpdates().should.equal(checkVal); + }); + + }); + + describe('#formatResults', () => { + + const FakeParseYaml = sinon.stub().returns({ test: { clients: {number: 7}}}); + ClientOrchestratorRewire.__set__('util.parseYaml', FakeParseYaml); + const myOrchestrator = new ClientOrchestratorRewire(); + + it('should group all client results into an array under a txStats label', () => { + const result0 = {result: [1] , start: new Date(2018, 11, 24, 10, 33), end: new Date(2018, 11, 24, 11, 33)}; + const result1 = {result: [2] , start: new Date(2018, 11, 24, 10, 34), end: new Date(2018, 11, 24, 11, 23)}; + const result2 = {result: [3] , start: new Date(2018, 11, 24, 10, 35), end: new Date(2018, 11, 24, 11, 13)}; + const testData = [result0, result1, result2]; + + const output = myOrchestrator.formatResults(testData); + output.txStats.should.deep.equal([1,2,3]); + }); + + it('should determine and persist the time when all clients have started', () => { + const compareStart = new Date(2018, 11, 24, 10, 35); + const result0 = {result: [1] , start: new Date(2018, 11, 24, 10, 33), end: new Date(2018, 11, 24, 11, 33)}; + const result1 = {result: [2] , start: new Date(2018, 11, 24, 10, 34), end: new Date(2018, 11, 24, 11, 13)}; + const result2 = {result: [3] , start: compareStart, end: new Date(2018, 11, 24, 11, 23)}; + const testData = [result0, result1, result2]; + + const output = myOrchestrator.formatResults(testData); + output.start.should.equal(compareStart); + }); + + it('should determine and persist the last time when all clients were running', () => { + const compareEnd = new Date(2018, 11, 24, 11, 13); + const result0 = {result: [1] , start: new Date(2018, 11, 24, 10, 33), end: new Date(2018, 11, 24, 11, 33)}; + const result1 = {result: [2] , start: new Date(2018, 11, 24, 10, 34), end: compareEnd}; + const result2 = {result: [3] , start: new Date(2018, 11, 24, 10, 35), end: new Date(2018, 11, 24, 11, 23)}; + const testData = [result0, result1, result2]; + + const output = myOrchestrator.formatResults(testData); + output.end.should.equal(compareEnd); + }); + }); + +}); diff --git a/packages/caliper-core/test/monitor/monitor-prometheus.js b/packages/caliper-core/test/monitor/monitor-prometheus.js new file mode 100644 index 000000000..5e8a2a9b3 --- /dev/null +++ b/packages/caliper-core/test/monitor/monitor-prometheus.js @@ -0,0 +1,175 @@ +/* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +'use strict'; + +const rewire = require('rewire'); +const PrometheusMonitorRewire = rewire('../../lib/monitor/monitor-prometheus'); + +const chai = require('chai'); +const should = chai.should(); +const sinon = require('sinon'); + +describe('Prometheus monitor implementation', () => { + + const fakePushClient = sinon.stub(); + const fakeQueryClient = sinon.stub(); + PrometheusMonitorRewire.__set__('PrometheusPushClient', fakePushClient); + PrometheusMonitorRewire.__set__('PrometheusQueryClient', fakeQueryClient); + + // Before/After + let clock; + beforeEach( () => { + clock = sinon.useFakeTimers(); + }); + + afterEach( () => { + clock.restore(); + }); + + // Test data + const ignore = { + metrics : { + ignore: ['prometheus', 'pushgateway', 'cadvisor'] + } + }; + + const includeOpts = { + Tag0: { + query: 'sum(rate(container_cpu_usage_seconds_total{name=~".+"}[$interval])) by (name) * 100', + statistic: 'average' + }, + Tag1: { + query: 'sum(rate(container_cpu_usage_seconds_total{name=~".+"}[$interval])) by (name) * 100', + statistic: 'maximum' + }, + Tag2: { + query: 'sum(container_memory_rss{name=~".+"}) by (name)', + statistic: 'average' + } + }; + + const include = { + metrics: { + include : includeOpts + } + }; + + describe('#constructor', () => { + + it('should set ignore list if provided', () => { + const mon = new PrometheusMonitorRewire(ignore); + mon.ignore.should.be.an('array').that.deep.equals(['prometheus', 'pushgateway', 'cadvisor']); + }); + + it('should not set ignore list if missing', () => { + const mon = new PrometheusMonitorRewire(include); + should.not.exist(mon.ignore); + }); + + it('should set include list if provided', () => { + const mon = new PrometheusMonitorRewire(include); + mon.include.should.be.an('object').that.deep.equals(includeOpts); + }); + + it('should not set include list if missing', () => { + const mon = new PrometheusMonitorRewire(ignore); + should.not.exist(mon.include); + }); + }); + + describe('#getPushClient', () => { + + it('should return the internal Push Client', () => { + const mon = new PrometheusMonitorRewire({}); + const test = 'penguin'; + mon.prometheusPushClient = test; + mon.getPushClient().should.equal(test); + }); + }); + + describe('#getQueryClient', ()=>{ + + it('should return the internal Query Client', () => { + const mon = new PrometheusMonitorRewire({}); + const test = 'penguin'; + mon.prometheusQueryClient = test; + mon.getQueryClient().should.equal(test); + }); + }); + + describe('#getPushGatewayURL', () => { + + it('should return the push gateway url from teh config file', () => { + const mon = new PrometheusMonitorRewire({push_url: '123'}); + mon.getPushGatewayURL().should.equal('123'); + }); + }); + + describe('#start', () => { + + it('should set the start time with the current time', () => { + clock.tick(42); + const mon = new PrometheusMonitorRewire({push_url: '123'}); + mon.start(); + mon.startTime.should.equal(0.042); + }); + + }); + + describe('#stop', () => { + it('should remove startTime if it exists', () => { + clock.tick(42); + const mon = new PrometheusMonitorRewire({push_url: '123'}); + mon.start(); + mon.startTime.should.equal(0.042); + mon.stop(); + should.not.exist(mon.startTime); + }); + }); + + describe('#restart', () => { + + it('should reset the start time', () => { + clock.tick(42); + const mon = new PrometheusMonitorRewire({push_url: '123'}); + mon.start(); + clock.tick(42); + mon.restart(); + mon.startTime.should.equal(0.084); + }); + + }); + + describe('#getResultColumnMapForQueryTag', () => { + + + it('should return a map with keys that correspond to the passed `include` keys, with default entries populated', () => { + + const mon = new PrometheusMonitorRewire(include); + const map = mon.getResultColumnMapForQueryTag('query', 'tag'); + + // Three keys + map.size.should.equal(3); + + // Tags, and Type should contain the corect information + map.get('Prometheus Query').should.equal('query'); + map.get('Name').should.equal('N/A'); + map.get('tag').should.equal('N/A'); + }); + }); + + + +}); diff --git a/packages/caliper-core/test/ndeep.js b/packages/caliper-core/test/ndeep.js deleted file mode 100644 index 25b4e1483..000000000 --- a/packages/caliper-core/test/ndeep.js +++ /dev/null @@ -1,37 +0,0 @@ -/* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -'use strict'; - -const bc = require('../lib/blockchain'); -const utils = require('../lib/utils/caliper-utils'); - - -// eslint-disable-next-line require-jsdoc -async function main(){ - - const blocky = new bc({bcType: 'demo'}); - const b2 = utils.flatten(blocky); - const res = utils.objToString(blocky,10); - console.log(res); - console.log(utils.objToString(b2,10)); - const b = blocky.toString(); - console.log(JSON.stringify(JSON.parse(b))); - -} - - -main(); - - diff --git a/packages/caliper-core/test/prometheus/prometheus-query-helper.js b/packages/caliper-core/test/prometheus/prometheus-query-helper.js new file mode 100644 index 000000000..58a0f3422 --- /dev/null +++ b/packages/caliper-core/test/prometheus/prometheus-query-helper.js @@ -0,0 +1,465 @@ +/* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +'use strict'; + +const PrometheusQueryHelper = require('../../lib/prometheus/prometheus-query-helper'); + +const chai = require('chai'); +chai.should(); + +describe('PrometheusQueryHelper implementation', () => { + + + describe('#buildStringRangeQuery', () => { + + const startTime = 100; + const endTime = 200; + const step = 45; + + it('should add start, end and step information', () => { + const demoString = 'test_the_heper'; + + const output = PrometheusQueryHelper.buildStringRangeQuery(demoString, startTime, endTime, step); + output.endsWith('&start=100&end=200&step=45').should.be.true; + }); + + it('should replace an instance of `{values}` with URI encoded version of that occurence', () => { + const demoString = 'sum(rate(container_cpu_usage_seconds_total{name=~".+"}[5m])) by (name) * 100'; + + const output = PrometheusQueryHelper.buildStringRangeQuery(demoString, startTime, endTime, step); + + output.should.not.contain('{'); + output.should.not.contain('}'); + output.should.contain(encodeURIComponent('{name=~".+"}')); + }); + + }); + + describe('#extractFirstValueFromQueryResponse', () => { + + + it('should deal with a vetor response that is numerical', () => { + const response = { + status:'success', + data:{ + resultType: 'vector', + result:[ + {metric: {name:'Name0'},value:[1565608080.458,'10']} + ] + }}; + const output = PrometheusQueryHelper.extractFirstValueFromQueryResponse(response, true); + output.should.equal(10); + }); + + it('should deal with a vetor response that is numerical string', () => { + const response = { + status:'success', + data:{ + resultType: 'vector', + result:[ + {metric: {name:'Name0'},value:[1565608080.458,'10']} + ] + }}; + const output = PrometheusQueryHelper.extractFirstValueFromQueryResponse(response); + output.should.equal(10); + }); + + it('should deal with a vetor response that is a string', () => { + const response = { + status:'success', + data:{ + resultType: 'vector', + result:[ + {metric: {name:'Name0'},value:[1565608080.458,'bob']} + ] + }}; + const output = PrometheusQueryHelper.extractFirstValueFromQueryResponse(response, false); + output.should.equal('bob'); + }); + + it('should deal with a matrix response that is numerical', () => { + const response = { + status:'success', + data:{ + resultType: 'matrix', + result:[ + {metric: {name:'Name0'},values:[[1565608080.458,3.1]]}, + {metric: {name:'Name1'},values:[[1565608080.458,0.2]]}, + {metric: {name:'Name2'},values:[[1565608080.458,0.1]]} + ] + }}; + const output = PrometheusQueryHelper.extractFirstValueFromQueryResponse(response, true); + output.should.be.an('map'); + output.size.should.equal(3); + output.get('Name0').should.equal(3.1); + output.get('Name1').should.equal(0.2); + output.get('Name2').should.equal(0.1); + }); + + it('should deal with a matrix response that contains numerical string values', () => { + const response = { + status:'success', + data:{ + resultType: 'matrix', + result:[ + {metric: {name:'Name0'},values:[[1565608080.458,'3.1']]}, + {metric: {name:'Name1'},values:[[1565608080.458,'0.2']]}, + {metric: {name:'Name2'},values:[[1565608080.458,'0.1']]} + ] + }}; + const output = PrometheusQueryHelper.extractFirstValueFromQueryResponse(response, true); + output.should.be.an('map'); + output.size.should.equal(3); + output.get('Name0').should.equal(3.1); + output.get('Name1').should.equal(0.2); + output.get('Name2').should.equal(0.1); + }); + + it('should deal with a matrix response that contains string values', () => { + const response = { + status:'success', + data:{ + resultType: 'matrix', + result:[ + {metric: {name:'Name0'},values:[[1565608080.458,'a']]}, + {metric: {name:'Name1'},values:[[1565608080.458,'b']]}, + {metric: {name:'Name2'},values:[[1565608080.458,'c']]} + ] + }}; + const output = PrometheusQueryHelper.extractFirstValueFromQueryResponse(response, false); + output.should.be.an('map'); + output.size.should.equal(3); + output.get('Name0').should.equal('a'); + output.get('Name1').should.equal('b'); + output.get('Name2').should.equal('c'); + }); + + it('should return `-` if passed too many results', () => { + const response = { + data: { + resultType: 'vector', + result : [1,2,3,4] + } + }; + const value = PrometheusQueryHelper.extractFirstValueFromQueryResponse(response); + value.should.equal('-'); + }); + + it('should return `-` if no value field' , () => { + const response = { + data: { + resultType: 'vector', + result : [{missing: 'yes'}] + } + }; + const value = PrometheusQueryHelper.extractFirstValueFromQueryResponse(response); + value.should.equal('-'); + }); + + it('should return contained value ', () => { + const response = { + data: { + resultType: 'vector', + result : [{value: [111, 1]}] + } + }; + const value = PrometheusQueryHelper.extractFirstValueFromQueryResponse(response); + value.should.equal(1); + }); + + it('should return parse contained value to float by default', () => { + const response = { + data: { + resultType: 'vector', + result : [{value: [111, '1']}] + } + }; + const value = PrometheusQueryHelper.extractFirstValueFromQueryResponse(response); + value.should.equal(1); + }); + + it('should return contained value as string if specified', () => { + const response = { + data: { + resultType: 'vector', + result : [{value: [111, '1']}] + } + }; + const value = PrometheusQueryHelper.extractFirstValueFromQueryResponse(response, false); + value.should.equal('1'); + }); + + it('should throw an error if unknown result type', () => { + (() => { + const response = { + data: { + resultType: 'penguin', + result : [{value: [111, '1']}] + } + }; + PrometheusQueryHelper.extractFirstValueFromQueryResponse(response, false); + }).should.throw(Error, /Unknown or missing result type: penguin/); + }); + + }); + + + describe('#extractStatisticFromRange', () => { + + it('should retrive the minimum value from a matrix response', () => { + const response = { + status:'success', + data:{ + resultType: 'matrix', + result:[ + {metric: {name:'Name0'},values:[[1565608080.458,3], [1565608080.458,4.8], [1565608080.458,0.2]]}, + {metric: {name:'Name1'},values:[[1565608080.458,1], [1565608080.458,8], [1565608080.458,6]]}, + {metric: {name:'Name2'},values:[[1565608080.458,0.1], [1565608080.458,-2], [1565608080.458,0.001]]} + ] + }}; + const output = PrometheusQueryHelper.extractStatisticFromRange(response,'min'); + output.should.be.an('map'); + output.size.should.equal(3); + output.get('Name0').should.equal(0.2); + output.get('Name1').should.equal(1); + output.get('Name2').should.equal(-2); + }); + + it('should retrive the minimum value from a matrix response that is numerical but contains no names', () => { + const response = { + status:'success', + data:{ + resultType: 'matrix', + result:[ + {values:[[1565608080.458,3], [1565608080.458,4.8], [1565608080.458,0.2]]} + ] + }}; + const output = PrometheusQueryHelper.extractStatisticFromRange(response,'min'); + output.should.be.an('map'); + output.size.should.equal(1); + output.get('unknown').should.equal(0.2); + }); + + it('should retrive the minimum value from a matrix response that contains a string', () => { + const response = { + status:'success', + data:{ + resultType: 'matrix', + result:[ + {metric: {name:'Name0'},values:[[1565608080.458,'banana'], [1565608080.458,'penguin'], [1565608080.458,'0.2']]}, + {metric: {name:'Name1'},values:[[1565608080.458,'1'], [1565608080.458,'bob'], [1565608080.458,'6']]}, + {metric: {name:'Name2'},values:[[1565608080.458,'sally'], [1565608080.458,'-2'], [1565608080.458,'0.001']]} + ] + }}; + const output = PrometheusQueryHelper.extractStatisticFromRange(response,'min'); + output.should.be.an('map'); + output.size.should.equal(3); + output.get('Name0').should.equal('-'); + output.get('Name1').should.equal('-'); + output.get('Name2').should.equal('-'); + }); + + it('should retrive the maximum value from a matrix response with a single value', () => { + const response = { + status:'success', + data:{ + resultType: 'matrix', + result:[ + { + metric: {name:'Name0'}, + values:[ + [1565608080.458,3] + ] + } + ] + }}; + const output = PrometheusQueryHelper.extractStatisticFromRange(response, 'max'); + output.should.be.an('map'); + output.size.should.equal(1); + output.get('Name0').should.equal(3); + }); + + it('should retrive the maximum value from a matrix response with multiple values', () => { + const response = { + status:'success', + data:{ + resultType: 'matrix', + result:[ + {metric: {name:'Name0'},values:[[1565608080.458,3], [1565608080.458,4.8], [1565608080.458,0.2]]}, + {metric: {name:'Name1'},values:[[1565608080.458,1], [1565608080.458,8], [1565608080.458,6]]}, + {metric: {name:'Name2'},values:[[1565608080.458,0.1], [1565608080.458,-2], [1565608080.458,0.001]]} + ] + }}; + const output = PrometheusQueryHelper.extractStatisticFromRange(response, 'max'); + output.should.be.an('map'); + output.size.should.equal(3); + output.get('Name0').should.equal(4.8); + output.get('Name1').should.equal(8); + output.get('Name2').should.equal(0.1); + }); + + const bob = { + 'status':'success', + 'data':{ + 'resultType':'matrix', + 'result':[ + { + 'metric':{}, + 'values':[ + [1565625618.564,'0.324'], + [1565625619.564,'0.324'], + [1565625620.564,'0.35']] + } + ] + } + }; + + + it('should retrive the maximum value from a matrix response with a single value and no name', () => { + const response = { + status:'success', + data:{ + resultType: 'matrix', + result:[ + { + values:[ + [1565608080.458,3] + ] + } + ] + }}; + const output = PrometheusQueryHelper.extractStatisticFromRange(response, 'max'); + output.should.be.an('map'); + output.size.should.equal(1); + output.get('unknown').should.equal(3); + }); + + it.only('should retrive the maximum value from a matrix response that is numerical but contains no names', () => { + const response = { + 'status':'success', + 'data':{ + 'resultType':'matrix', + 'result':[ + { + 'metric':{}, + 'values':[ + [1565625618.564,'0.7'], + [1565625619.564,'0.324'], + [1565625620.564,'0.35']] + } + ] + } + }; + const output = PrometheusQueryHelper.extractStatisticFromRange(response, 'max'); + output.should.be.an('map'); + output.size.should.equal(1); + output.get('unknown').should.equal(0.7); + }); + + it('should retrive the maximum value from a matrix response that is string', () => { + const response = { + status:'success', + data:{ + resultType: 'matrix', + result:[ + {metric: {name:'Name0'},values:[[1565608080.458,'banana'], [1565608080.458,'penguin'], [1565608080.458,'0.2']]}, + {metric: {name:'Name1'},values:[[1565608080.458,'1'], [1565608080.458,'bob'], [1565608080.458,'6']]}, + {metric: {name:'Name2'},values:[[1565608080.458,'sally'], [1565608080.458,'-2'], [1565608080.458,'0.001']]} + ] + }}; + const output = PrometheusQueryHelper.extractStatisticFromRange(response, 'max'); + output.should.be.an('map'); + output.size.should.equal(3); + output.get('Name0').should.equal('-'); + output.get('Name1').should.equal('-'); + output.get('Name2').should.equal('-'); + }); + + it('should retrieve the average value from a matrix response', () => { + const repsonse = { + data: { + resultType: 'matrix', + result : [{values: [[111, 1] , [111, 1], [111, 2], [111, 2], [111, 8]]}] + } + }; + const output = PrometheusQueryHelper.extractStatisticFromRange(repsonse, 'avg'); + output.should.be.an('map'); + output.size.should.equal(1); + output.get('unknown').should.equal(2.8); + }); + + it('should return `-` if passed too few results', () => { + const response = { + data: { + resultType: 'matrix', + result : [1] + } + }; + const output = PrometheusQueryHelper.extractStatisticFromRange(response,'min'); + output.should.be.an('map'); + output.size.should.equal(1); + output.get('unknown').should.equal('-'); + }); + + it('should return `-` if no values field' , () => { + const response = { + data: { + resultType: 'matrix', + result : [{missing: 'yes'}] + } + }; + const output = PrometheusQueryHelper.extractStatisticFromRange(response,'min'); + output.should.be.an('map'); + output.size.should.equal(1); + output.get('unknown').should.equal('-'); + }); + + }); + + + describe('#extractValuesFromTimeSeries', () => { + + const numberSeries = [ + [111, 1], + [111, 2], + [111, 3], + [111, 4] + ]; + + const stringSeries = [ + [111, '1'], + [111, '2'], + [111, '3'], + [111, '4'] + ]; + + it('should return numeric values by default', () => { + const vals = PrometheusQueryHelper.extractValuesFromTimeSeries(numberSeries); + vals.should.be.an('array').of.length(4).that.deep.equals([1,2,3,4]); + }); + + it('should cast to numeric values if specified', () => { + const vals = PrometheusQueryHelper.extractValuesFromTimeSeries(stringSeries, true); + vals.should.be.an('array').of.length(4).that.deep.equals([1,2,3,4]); + }); + + it('should return string values if specified', () => { + const vals = PrometheusQueryHelper.extractValuesFromTimeSeries(stringSeries, false); + vals.should.be.an('array').of.length(4).that.deep.equals(['1','2','3','4']); + }); + }); + +}); + diff --git a/packages/caliper-core/test/report/report.js b/packages/caliper-core/test/report/report.js new file mode 100644 index 000000000..22a6ab794 --- /dev/null +++ b/packages/caliper-core/test/report/report.js @@ -0,0 +1,171 @@ +/* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +'use strict'; + +const Report = require('../../lib/report/report'); + +const chai = require('chai'); +chai.should(); +const sinon = require('sinon'); + +describe('report implementation', () => { + + describe('#getLocalResultValues', () => { + + it('should retrieve a result column map', () => { + const report = new Report(); + + const getResultColumnMapSpy = new sinon.stub().returns(new Map()); + report.getResultColumnMap = getResultColumnMapSpy; + + report.getLocalResultValues('label', {}); + + sinon.assert.calledOnce(getResultColumnMapSpy); + }); + + it('should set Name to `unknown` if missing', () => { + const report = new Report(); + const output = report.getLocalResultValues(null, {}); + output.get('Name').should.equal('unknown'); + }); + + it('should set Name if available', () => { + const report = new Report(); + const output = report.getLocalResultValues('myTestLabel', {}); + output.get('Name').should.equal('myTestLabel'); + }); + + it('should set Succ to `-` if missing', () => { + const report = new Report(); + const output = report.getLocalResultValues('myTestLabel', {}); + output.get('Succ').should.equal('-'); + }); + + it('should set Succ if available', () => { + const report = new Report(); + const output = report.getLocalResultValues('myTestLabel', {succ: '42'}); + output.get('Succ').should.equal('42'); + }); + + it('should set Succ to zero if passed', () => { + const report = new Report(); + const output = report.getLocalResultValues('myTestLabel', {succ: '0'}); + output.get('Succ').should.equal('0'); + }); + + it('should set Fail to `-` if missing', () => { + const report = new Report(); + const output = report.getLocalResultValues('myTestLabel', {}); + output.get('Fail').should.equal('-'); + }); + + it('should set Fail if available', () => { + const report = new Report(); + const output = report.getLocalResultValues('myTestLabel', {fail: '38'}); + output.get('Fail').should.equal('38'); + }); + + it('should set Max Latency to `-` if missing', () => { + const report = new Report(); + const output = report.getLocalResultValues('myTestLabel', {}); + output.get('Max Latency (s)').should.equal('-'); + }); + + it('should set Max Latency to 2DP if available', () => { + const report = new Report(); + const output = report.getLocalResultValues('myTestLabel', {delay: { max: 1.2322}} ); + output.get('Max Latency (s)').should.equal('1.23'); + }); + it('should set Min Latency to `-` if missing', () => { + const report = new Report(); + const output = report.getLocalResultValues('myTestLabel', {}); + output.get('Min Latency (s)').should.equal('-'); + }); + + it('should set Min Latency to 2DP if available', () => { + const report = new Report(); + const output = report.getLocalResultValues('myTestLabel', {delay: { min: 0.2322}}); + output.get('Min Latency (s)').should.equal('0.23'); + }); + + it('should set Avg Latency to `-` if missing', () => { + const report = new Report(); + const output = report.getLocalResultValues('myTestLabel', {}); + output.get('Avg Latency (s)').should.equal('-'); + }); + + it('should set Avg Latency to 2DP if available', () => { + const report = new Report(); + const output = report.getLocalResultValues('myTestLabel', { succ: 3, delay: { sum: 10}}); + output.get('Avg Latency (s)').should.equal('3.33'); + }); + + it('should set Send Rate `-` if missing', () => { + const report = new Report(); + const output = report.getLocalResultValues('myTestLabel', {}); + output.get('Send Rate (TPS)').should.equal('-'); + }); + + it('should set Send Rate to 1DP if available', () => { + const report = new Report(); + const output = report.getLocalResultValues('myTestLabel', { succ:500, fail:0, create:{min:1565001755.094, max:1565001774.893} }); + output.get('Send Rate (TPS)').should.equal('25.3'); + }); + + it('should set Throughput to `-` if missing', () => { + const report = new Report(); + const output = report.getLocalResultValues('myTestLabel', {}); + output.get('Throughput (TPS)').should.equal('-'); + }); + + it('should set Throughput to 1DP if available', () => { + const report = new Report(); + const output = report.getLocalResultValues('myTestLabel', {succ:500,fail:0,create:{min:1565001755.094,max:1565001774.893},final:{min:1565001755.407,max:1565001774.988,last:1565001774.988},delay:{min:0.072,max:0.342,sum:98.64099999999999,detail:[]},out:[],sTPTotal:0,sTTotal:0,invokeTotal:0,length:500}); + output.get('Throughput (TPS)').should.equal('25.1'); + }); + }); + + describe('#convertToTable', () => { + + it('should handle a single result map', () => { + const report = new Report(); + const results = new Map(); + results.set('Val1', 1); + results.set('Val2', 2); + + const table = report.convertToTable(results); + table.should.be.an('array').with.length(2); + table[0].should.be.an('array').that.contains('Val1','Val2'); + table[1].should.be.an('array').that.contains(1,2); + }); + + it('should handle an array of result maps', () => { + const report = new Report(); + const results1 = new Map(); + results1.set('Val1', 1); + results1.set('Val2', 2); + const results2 = new Map(); + results2.set('Val1', 3); + results2.set('Val2', 4); + + const table = report.convertToTable([results1, results2]); + table.should.be.an('array').with.length(3); + table[0].should.be.an('array').that.contains('Val1','Val2'); + table[1].should.be.an('array').that.contains(1,2); + table[2].should.be.an('array').that.contains(3,4); + }); + }); + +}); diff --git a/packages/caliper-fabric/lib/fabric.js b/packages/caliper-fabric/lib/fabric.js index 616631781..5578df673 100644 --- a/packages/caliper-fabric/lib/fabric.js +++ b/packages/caliper-fabric/lib/fabric.js @@ -366,7 +366,7 @@ class Fabric extends BlockchainInterface { // resolve the failed transaction with the current time and error message resolve({ successful: false, - message: `Commit timeout on ${eventSource.peer}`, + message: `Commit timeout on ${eventSource.peer} for transaction ${txId}`, time: time }); }, this._getRemainingTimeout(startTime, timeout)); @@ -1729,7 +1729,6 @@ class Fabric extends BlockchainInterface { // the exception should propagate up for an invalid channel name, indicating a user callback module error let channel = invoker.getChannel(querySettings.channel, true); - if (countAsLoad && context.engine) { context.engine.submitCallback(1); } diff --git a/packages/caliper-fabric/package.json b/packages/caliper-fabric/package.json index 2d6c3fec5..d08ed0ea2 100644 --- a/packages/caliper-fabric/package.json +++ b/packages/caliper-fabric/package.json @@ -1,5 +1,6 @@ { "name": "@hyperledger/caliper-fabric", + "description": "Hyperledger Fabric adaptor for Caliper, enabling the running of a performance benchmarks that interact with Fabric", "version": "0.1.0", "repository": { "type": "git", diff --git a/packages/caliper-iroha/package.json b/packages/caliper-iroha/package.json index f6c1e728f..eacdd50f1 100644 --- a/packages/caliper-iroha/package.json +++ b/packages/caliper-iroha/package.json @@ -1,5 +1,6 @@ { "name": "@hyperledger/caliper-iroha", + "description": "Hyperledger Iroha adaptor for Caliper, enabling the running of a performance benchmarks that interact with Iroha", "version": "0.1.0", "repository": { "type": "git", diff --git a/packages/caliper-samples/benchmark/composer/config-composer-pid.yaml b/packages/caliper-samples/benchmark/composer/config-composer-pid.yaml index 615a4e982..a75d13bca 100644 --- a/packages/caliper-samples/benchmark/composer/config-composer-pid.yaml +++ b/packages/caliper-samples/benchmark/composer/config-composer-pid.yaml @@ -42,6 +42,9 @@ test: arguments: testAssets: 50 callback: benchmark/composer/composer-samples/basic-sample-network.js +observer: + interval: 1 + type: local monitor: type: - docker @@ -53,4 +56,4 @@ monitor: - command: node arguments: local-client.js multiOutput: avg - interval: 1 \ No newline at end of file + interval: 1 diff --git a/packages/caliper-samples/benchmark/composer/config.yaml b/packages/caliper-samples/benchmark/composer/config.yaml index 62c889198..cc91ee828 100644 --- a/packages/caliper-samples/benchmark/composer/config.yaml +++ b/packages/caliper-samples/benchmark/composer/config.yaml @@ -42,6 +42,9 @@ test: arguments: testAssets: 50 callback: benchmark/composer/composer-samples/basic-sample-network.js +observer: + interval: 1 + type: local monitor: type: - docker diff --git a/packages/caliper-samples/benchmark/drm/config.yaml b/packages/caliper-samples/benchmark/drm/config.yaml index 5c32e5a50..059f7f2c3 100644 --- a/packages/caliper-samples/benchmark/drm/config.yaml +++ b/packages/caliper-samples/benchmark/drm/config.yaml @@ -38,6 +38,9 @@ test: opts: tps: 1 callback: benchmark/drm/query.js +observer: + interval: 1 + type: local monitor: type: docker docker: @@ -47,4 +50,4 @@ monitor: - peer0.org2.example.com - peer1.org2.example.com - orderer.example.com - interval: 1 \ No newline at end of file + interval: 1 diff --git a/packages/caliper-samples/benchmark/marbles/config.yaml b/packages/caliper-samples/benchmark/marbles/config.yaml index 34a84fac3..28585f81a 100644 --- a/packages/caliper-samples/benchmark/marbles/config.yaml +++ b/packages/caliper-samples/benchmark/marbles/config.yaml @@ -16,28 +16,19 @@ test: clients: type: local - number: 5 + number: 2 rounds: - label: init txNumber: - - 500 - - 500 - - 500 + - 250 rateControl: - type: fixed-rate opts: tps: 25 - - type: fixed-rate - opts: - tps: 50 - - type: fixed-rate - opts: - tps: 75 callback: benchmark/marbles/init.js - - label: query + - label: init2 txNumber: - - 15 - - 15 + - 250 rateControl: - type: fixed-rate opts: @@ -45,8 +36,12 @@ test: - type: fixed-rate opts: tps: 5 - callback: benchmark/marbles/query.js + callback: benchmark/marbles/init.js +observer: + interval: 1 + type: local monitor: + interval: 1 type: - docker - process @@ -55,6 +50,5 @@ monitor: - all process: - command: node - arguments: local-client.js + arguments: fabricClientWorker.js multiOutput: avg - interval: 1 diff --git a/packages/caliper-samples/benchmark/simple/config-composite-rate.yaml b/packages/caliper-samples/benchmark/simple/config-composite-rate.yaml index fcf455fa4..0ff7ed476 100644 --- a/packages/caliper-samples/benchmark/simple/config-composite-rate.yaml +++ b/packages/caliper-samples/benchmark/simple/config-composite-rate.yaml @@ -48,6 +48,9 @@ test: arguments: money: 10000 callback: benchmark/simple/open.js +observer: + interval: 1 + type: local monitor: type: - docker @@ -59,4 +62,4 @@ monitor: - command: node arguments: local-client.js multiOutput: avg - interval: 1 \ No newline at end of file + interval: 1 diff --git a/packages/caliper-samples/benchmark/simple/config-feedback-rate.yaml b/packages/caliper-samples/benchmark/simple/config-feedback-rate.yaml index 67369e689..780f1e79b 100644 --- a/packages/caliper-samples/benchmark/simple/config-feedback-rate.yaml +++ b/packages/caliper-samples/benchmark/simple/config-feedback-rate.yaml @@ -48,6 +48,9 @@ test: opts: tps: 400 callback: benchmark/simple/query.js +observer: + interval: 1 + type: local monitor: type: - docker diff --git a/packages/caliper-samples/benchmark/simple/config-iroha.yaml b/packages/caliper-samples/benchmark/simple/config-iroha.yaml index 3266e6f74..fc6f1e556 100644 --- a/packages/caliper-samples/benchmark/simple/config-iroha.yaml +++ b/packages/caliper-samples/benchmark/simple/config-iroha.yaml @@ -37,6 +37,9 @@ test: opts: tps: 50 callback: benchmark/simple/query.js +observer: + interval: 1 + type: local monitor: type: - docker diff --git a/packages/caliper-samples/benchmark/simple/config-linear-rate.yaml b/packages/caliper-samples/benchmark/simple/config-linear-rate.yaml index 85445e1d8..c0da8ab0f 100644 --- a/packages/caliper-samples/benchmark/simple/config-linear-rate.yaml +++ b/packages/caliper-samples/benchmark/simple/config-linear-rate.yaml @@ -32,6 +32,9 @@ test: arguments: money: 10000 callback: benchmark/simple/open.js +observer: + interval: 1 + type: local monitor: type: - docker @@ -43,4 +46,4 @@ monitor: - command: node arguments: local-client.js multiOutput: avg - interval: 1 \ No newline at end of file + interval: 1 diff --git a/packages/caliper-samples/benchmark/simple/config-record-replay-rate.yaml b/packages/caliper-samples/benchmark/simple/config-record-replay-rate.yaml index 5c838594f..7a07d3a4f 100644 --- a/packages/caliper-samples/benchmark/simple/config-record-replay-rate.yaml +++ b/packages/caliper-samples/benchmark/simple/config-record-replay-rate.yaml @@ -78,6 +78,9 @@ test: arguments: money: 10000 callback: benchmark/simple/open.js +observer: + interval: 1 + type: local monitor: type: - docker @@ -89,4 +92,4 @@ monitor: - command: node arguments: local-client.js multiOutput: avg - interval: 1 \ No newline at end of file + interval: 1 diff --git a/packages/caliper-samples/benchmark/simple/config-sawtooth-feedback.yaml b/packages/caliper-samples/benchmark/simple/config-sawtooth-feedback.yaml index 9e24083ed..27281f957 100644 --- a/packages/caliper-samples/benchmark/simple/config-sawtooth-feedback.yaml +++ b/packages/caliper-samples/benchmark/simple/config-sawtooth-feedback.yaml @@ -41,6 +41,9 @@ test: opts: tps: 1 callback: benchmark/simple/query.js +observer: + interval: 1 + type: local monitor: type: docker docker: diff --git a/packages/caliper-samples/benchmark/simple/config-sawtooth.yaml b/packages/caliper-samples/benchmark/simple/config-sawtooth.yaml index f4b6e40ec..948014278 100644 --- a/packages/caliper-samples/benchmark/simple/config-sawtooth.yaml +++ b/packages/caliper-samples/benchmark/simple/config-sawtooth.yaml @@ -39,6 +39,9 @@ test: opts: tps: 1 callback: benchmark/simple/query.js +observer: + interval: 1 + type: local monitor: type: docker docker: diff --git a/packages/caliper-samples/benchmark/simple/config-zookeeper.yaml b/packages/caliper-samples/benchmark/simple/config-zookeeper.yaml index 195b31f19..f41d9366e 100644 --- a/packages/caliper-samples/benchmark/simple/config-zookeeper.yaml +++ b/packages/caliper-samples/benchmark/simple/config-zookeeper.yaml @@ -41,6 +41,9 @@ test: opts: tps: 200 callback: benchmark/simple/query.js +observer: + interval: 1 + type: local monitor: type: - docker diff --git a/packages/caliper-samples/benchmark/simple/config.yaml b/packages/caliper-samples/benchmark/simple/config.yaml index 92c49c506..8274e5b35 100644 --- a/packages/caliper-samples/benchmark/simple/config.yaml +++ b/packages/caliper-samples/benchmark/simple/config.yaml @@ -52,6 +52,9 @@ test: arguments: money: 100 callback: benchmark/simple/transfer.js +observer: + interval: 1 + type: local monitor: type: - docker diff --git a/packages/caliper-samples/benchmark/simple/config_linear.yaml b/packages/caliper-samples/benchmark/simple/config_linear.yaml index 55217f5d8..5e35a48f8 100644 --- a/packages/caliper-samples/benchmark/simple/config_linear.yaml +++ b/packages/caliper-samples/benchmark/simple/config_linear.yaml @@ -119,6 +119,9 @@ test: opts: tps: 200 callback: benchmark/simple/query.js +observer: + interval: 1 + type: local monitor: type: - docker diff --git a/packages/caliper-samples/benchmark/simple/config_long.yaml b/packages/caliper-samples/benchmark/simple/config_long.yaml index 79cee4b36..e31dfe075 100644 --- a/packages/caliper-samples/benchmark/simple/config_long.yaml +++ b/packages/caliper-samples/benchmark/simple/config_long.yaml @@ -43,6 +43,9 @@ test: opts: tps: 200 callback: benchmark/simple/query.js +observer: + interval: 1 + type: local monitor: type: - docker diff --git a/packages/caliper-samples/benchmark/smallbank/config-smallbank-sawtooth.yaml b/packages/caliper-samples/benchmark/smallbank/config-smallbank-sawtooth.yaml index fbc9bae6a..dbb322564 100644 --- a/packages/caliper-samples/benchmark/smallbank/config-smallbank-sawtooth.yaml +++ b/packages/caliper-samples/benchmark/smallbank/config-smallbank-sawtooth.yaml @@ -39,6 +39,9 @@ test: opts: tps: 50 callback: benchmark/smallbank/query.js +observer: + interval: 1 + type: local monitor: type: docker docker: diff --git a/packages/caliper-samples/benchmark/smallbank/config.yaml b/packages/caliper-samples/benchmark/smallbank/config.yaml index d0fbea0ee..4aae9fa62 100644 --- a/packages/caliper-samples/benchmark/smallbank/config.yaml +++ b/packages/caliper-samples/benchmark/smallbank/config.yaml @@ -39,6 +39,9 @@ test: opts: tps: 50 callback: benchmark/smallbank/query.js +observer: + interval: 1 + type: local monitor: type: docker docker: diff --git a/packages/caliper-samples/benchmark/smallbank/config_linear.yaml b/packages/caliper-samples/benchmark/smallbank/config_linear.yaml index 05e3f1b2b..a7b8ff3dc 100644 --- a/packages/caliper-samples/benchmark/smallbank/config_linear.yaml +++ b/packages/caliper-samples/benchmark/smallbank/config_linear.yaml @@ -115,6 +115,9 @@ test: opts: tps: 50 callback: benchmark/smallbank/query.js +observer: + interval: 1 + type: local monitor: type: docker docker: diff --git a/packages/caliper-samples/benchmark/smallbank/config_long.yaml b/packages/caliper-samples/benchmark/smallbank/config_long.yaml index eeef8e9ef..ef8f18904 100644 --- a/packages/caliper-samples/benchmark/smallbank/config_long.yaml +++ b/packages/caliper-samples/benchmark/smallbank/config_long.yaml @@ -39,6 +39,9 @@ test: opts: tps: 500 callback: benchmark/smallbank/query.js +observer: + interval: 1 + type: local monitor: type: docker docker: diff --git a/packages/caliper-samples/network/fabric-v1.4/2org1peergoleveldb/fabric-go.yaml b/packages/caliper-samples/network/fabric-v1.4/2org1peergoleveldb/fabric-go.yaml index bd8cda0a9..e222643cd 100644 --- a/packages/caliper-samples/network/fabric-v1.4/2org1peergoleveldb/fabric-go.yaml +++ b/packages/caliper-samples/network/fabric-v1.4/2org1peergoleveldb/fabric-go.yaml @@ -10,7 +10,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# +# name: Fabric version: "1.0" diff --git a/packages/caliper-sawtooth/package.json b/packages/caliper-sawtooth/package.json index 9ed1e1f37..cbaf6e3d0 100644 --- a/packages/caliper-sawtooth/package.json +++ b/packages/caliper-sawtooth/package.json @@ -1,5 +1,6 @@ { "name": "@hyperledger/caliper-sawtooth", + "description": "Hyperledger Sawtooth adaptor for Caliper, enabling the running of a performance benchmarks that interact with Sawtooth", "version": "0.1.0", "repository": { "type": "git",