diff --git a/packages/caliper-cli/caliper.js b/packages/caliper-cli/caliper.js index 58c0e276a8..d397e59791 100644 --- a/packages/caliper-cli/caliper.js +++ b/packages/caliper-cli/caliper.js @@ -16,9 +16,7 @@ 'use strict'; process.env.SUPPRESS_NO_CONFIG_WARNING = true; -const cmdUtil = require('./lib/utils/cmdutils'); const yargs = require('yargs'); -const chalk = require('chalk'); const version = 'v' + require('./package.json').version; let results = yargs @@ -35,11 +33,7 @@ let results = yargs .argv; results.thePromise.then( () => { - if (!results.quiet) { - cmdUtil.log(chalk.green('\nCommand succeeded\n')); - } process.exit(0); }).catch((error) => { - cmdUtil.log(error.stack+chalk.red('\nCommand failed\n')); process.exit(1); }); diff --git a/packages/caliper-cli/lib/benchmark/lib/runBenchmark.js b/packages/caliper-cli/lib/benchmark/lib/runBenchmark.js index 33331e8e54..3b4e29aa87 100644 --- a/packages/caliper-cli/lib/benchmark/lib/runBenchmark.js +++ b/packages/caliper-cli/lib/benchmark/lib/runBenchmark.js @@ -15,8 +15,7 @@ 'use strict'; const {CaliperEngine, CaliperUtils, ConfigUtil} = require('@hyperledger/caliper-core'); -const chalk = require('chalk'); -const cmdUtil = require('../../utils/cmdutils'); +const logger = CaliperUtils.getLogger('CLI'); const path = require('path'); const fs = require('fs'); /** @@ -30,47 +29,66 @@ class RunBenchmark { * @param {string} argv argument list from caliper benchmark command */ static async handler(argv) { - let workspace = ConfigUtil.get(ConfigUtil.keys.Workspace, './'); - let benchConfigFile = ConfigUtil.get(ConfigUtil.keys.BenchConfig, undefined); - let blockchainConfigFile = ConfigUtil.get(ConfigUtil.keys.NetworkConfig, undefined); + let workspacePath = ConfigUtil.get(ConfigUtil.keys.Workspace); + let benchmarkConfigPath = ConfigUtil.get(ConfigUtil.keys.BenchConfig); + let networkConfigPath = ConfigUtil.get(ConfigUtil.keys.NetworkConfig); // Workspace is expected to be the root location of working folders - workspace = path.resolve(workspace); - benchConfigFile = path.isAbsolute(benchConfigFile) ? benchConfigFile : path.join(workspace, benchConfigFile); - blockchainConfigFile = path.isAbsolute(blockchainConfigFile) ? blockchainConfigFile : path.join(workspace, blockchainConfigFile); + workspacePath = path.resolve(workspacePath); + benchmarkConfigPath = CaliperUtils.resolvePath(benchmarkConfigPath, workspacePath); + networkConfigPath = CaliperUtils.resolvePath(networkConfigPath, workspacePath); - if(!benchConfigFile || !fs.existsSync(benchConfigFile)) { - throw new Error(`Benchmark configuration file "${benchConfigFile || 'UNSET'}" does not exist`); + if(!benchmarkConfigPath || !fs.existsSync(benchmarkConfigPath)) { + let msg = `Benchmark configuration file "${benchmarkConfigPath || 'UNSET'}" does not exist`; + logger.error(msg); + throw new Error(msg); } - if(!blockchainConfigFile || !fs.existsSync(blockchainConfigFile)) { - throw new Error(`Network configuration file "${blockchainConfigFile || 'UNSET'}" does not exist`); + if(!networkConfigPath || !fs.existsSync(networkConfigPath)) { + let msg = `Network configuration file "${networkConfigPath || 'UNSET'}" does not exist`; + logger.error(msg); + throw new Error(msg); } + let benchmarkConfig = CaliperUtils.parseYaml(benchmarkConfigPath); + let networkConfig = CaliperUtils.parseYaml(networkConfigPath); + let blockchainType = ''; - let networkObject = CaliperUtils.parseYaml(blockchainConfigFile); - if (networkObject.hasOwnProperty('caliper') && networkObject.caliper.hasOwnProperty('blockchain')) { - blockchainType = networkObject.caliper.blockchain; + if (networkConfig.caliper && networkConfig.caliper.blockchain) { + blockchainType = networkConfig.caliper.blockchain; } else { - throw new Error('The configuration file [' + blockchainConfigFile + '] is missing its "caliper.blockchain" attribute'); + let msg = `Network configuration file "${networkConfigPath}" is missing its "caliper.blockchain" attribute`; + logger.error(msg); + throw new Error(msg); } + let knownError = false; + try { - cmdUtil.log(chalk.blue.bold('Benchmark for target Blockchain type ' + blockchainType + ' about to start')); - const {AdminClient, ClientFactory} = require('@hyperledger/caliper-' + blockchainType); - const adminClient = new AdminClient(blockchainConfigFile, workspace); - const clientFactory = new ClientFactory(blockchainConfigFile, workspace); + logger.info(`Set workspace path: ${workspacePath}`); + logger.info(`Set benchmark configuration path: ${benchmarkConfigPath}`); + logger.info(`Set network configuration path: ${networkConfigPath}`); + logger.info(`Detected SUT type: ${blockchainType}`); - const engine = new CaliperEngine(benchConfigFile, blockchainConfigFile, adminClient, clientFactory, workspace); + const {AdminClient, ClientFactory} = require(`@hyperledger/caliper-${blockchainType}`); + const blockchainAdapter = new AdminClient(networkConfigPath, workspacePath); + const workerFactory = new ClientFactory(networkConfigPath, workspacePath); + + const engine = new CaliperEngine(benchmarkConfig, networkConfig, blockchainAdapter, workerFactory); const response = await engine.run(); if (response === 0) { - cmdUtil.log(chalk.blue.bold('Benchmark run successful')); + logger.info('Benchmark successfully finished'); } else { - cmdUtil.log(chalk.red.bold(`Benchmark failure with code ${response}`)); - throw new Error(`Benchmark failure with code ${response}`); + knownError = true; + let msg = `Benchmark failed with error code ${response}`; + logger.error(msg); + throw new Error(msg); } } catch (err) { + if (!knownError) { + logger.error(`Unexpected error during benchmark execution: ${err.stack || err}`); + } throw err; } } diff --git a/packages/caliper-cli/lib/bind/bind.js b/packages/caliper-cli/lib/bind/bind.js index aa7280724c..1e521172f1 100644 --- a/packages/caliper-cli/lib/bind/bind.js +++ b/packages/caliper-cli/lib/bind/bind.js @@ -15,7 +15,6 @@ 'use strict'; const { CaliperUtils, ConfigUtil } = require('@hyperledger/caliper-core'); -const chalk = require('chalk'); const cmdUtil = require('../utils/cmdutils'); const path = require('path'); @@ -107,7 +106,7 @@ class Bind { settings = {}; } - cmdUtil.log(chalk.blue.bold(`Binding for ${sut}@${sdk}. This might take some time...`)); + logger.info(`Binding for ${sut}@${sdk}. This might take some time...`); try { // combine, then convert the arguments to an array let npmArgs; diff --git a/packages/caliper-cli/lib/utils/cmdutils.js b/packages/caliper-cli/lib/utils/cmdutils.js index 049ccb4eaf..84f3c45db4 100644 --- a/packages/caliper-cli/lib/utils/cmdutils.js +++ b/packages/caliper-cli/lib/utils/cmdutils.js @@ -23,18 +23,6 @@ const { spawn }= require('child_process'); */ class CmdUtil { - - /** Simple log method to output to the console - * Used to put a single console.log() here, so eslinting is easier. - * And if this needs to written to a file at some point it is also eaiser - */ - static log(){ - Array.from(arguments).forEach((s)=>{ - // eslint-disable-next-line no-console - console.log(s); - }); - } - /** * Invokes a given command in a spawned child process and attaches all standard IO. * @param {string} cmd The command to be run. diff --git a/packages/caliper-cli/package.json b/packages/caliper-cli/package.json index ab80ccac3c..289eccb404 100644 --- a/packages/caliper-cli/package.json +++ b/packages/caliper-cli/package.json @@ -31,7 +31,6 @@ "@hyperledger/caliper-sawtooth": "0.2.0", "@hyperledger/caliper-ethereum": "0.2.0", "@hyperledger/caliper-fisco-bcos": "0.2.0", - "chalk": "1.1.3", "yargs": "10.0.3" }, "devDependencies": { diff --git a/packages/caliper-core/lib/common/utils/caliper-utils.js b/packages/caliper-core/lib/common/utils/caliper-utils.js index db1580704e..28d9db637d 100644 --- a/packages/caliper-core/lib/common/utils/caliper-utils.js +++ b/packages/caliper-core/lib/common/utils/caliper-utils.js @@ -64,7 +64,7 @@ class CaliperUtils { * @param {String} root_path root path to use * @return {String} The resolved absolute path. */ - static resolvePath(relOrAbsPath, root_path) { + static resolvePath(relOrAbsPath, root_path = undefined) { if (!relOrAbsPath) { throw new Error('Util.resolvePath: Parameter is undefined'); } @@ -73,7 +73,7 @@ class CaliperUtils { return relOrAbsPath; } - return path.resolve(root_path, relOrAbsPath); + return path.resolve(root_path || Config.get(Config.keys.Workspace), relOrAbsPath); } /** diff --git a/packages/caliper-core/lib/master/caliper-engine.js b/packages/caliper-core/lib/master/caliper-engine.js index ebef570558..c1a5803297 100644 --- a/packages/caliper-core/lib/master/caliper-engine.js +++ b/packages/caliper-core/lib/master/caliper-engine.js @@ -16,6 +16,7 @@ const Blockchain = require('../common/core/blockchain'); const CaliperUtils = require('../common/utils/caliper-utils'); +const ConfigUtils = require('../common/config/config-util'); const RoundOrchestrator = require('./test-runners/round-orchestrator'); const BenchValidator = require('../common/utils/benchmark-validator'); @@ -26,32 +27,30 @@ const logger = CaliperUtils.getLogger('caliper-engine'); */ class CaliperEngine { /** - * - * @param {String} absConfigFile fully qualified path of the test configuration file - * @param {String} absNetworkFile fully qualified path of the blockchain configuration file - * @param {AdminClient} admin a constructed Caliper Admin Client - * @param {ClientFactory} clientFactory a Caliper Client Factory - * @param {String} workspace fully qualified path to the root location of network files + * Initializes the CaliperEngine instance. + * @param {object} benchmarkConfig The benchmark configuration object. + * @param {object} networkConfig The network configuration object. + * @param {AdminClient} blockchainAdapter The blockchain adapter instance. + * @param {ClientFactory} workerFactory The worker process factory. */ - constructor(absConfigFile, absNetworkFile, admin, clientFactory, workspace) { - this.absConfigFile = absConfigFile; - this.absNetworkFile = absNetworkFile; - this.admin = admin; - this.clientFactory = clientFactory; - this.workspace = workspace; + constructor(benchmarkConfig, networkConfig, blockchainAdapter, workerFactory) { + this.benchmarkConfig = benchmarkConfig; + this.networkConfig = networkConfig; + this.blockchainAdapter = blockchainAdapter; + this.workerFactory = workerFactory; + this.workspace = ConfigUtils.get(ConfigUtils.keys.workspace); this.returnCode = -1; } /** * Executes the given start/end command with proper checking and error handling. - * @param {object} networkConfig The network configuration object. //TODO: move this to instance var * @param {string} commandName Either "start" or "end". Used in the error messages. * @param {number} errorStatusStart The last taken error status code. Execution errors will use the next 3 status code. * @private */ - async _executeCommand(networkConfig, commandName, errorStatusStart) { - if (networkConfig.caliper && networkConfig.caliper.command && networkConfig.caliper.command[commandName]) { - let command = networkConfig.caliper.command[commandName]; + async _executeCommand(commandName, errorStatusStart) { + if (this.networkConfig.caliper && this.networkConfig.caliper.command && this.networkConfig.caliper.command[commandName]) { + let command = this.networkConfig.caliper.command[commandName]; if (typeof command !== 'string') { let msg = `Network configuration attribute "caliper.command.${commandName}" is not a string`; logger.error(msg, command); @@ -64,6 +63,7 @@ class CaliperEngine { this.returnCode = errorStatusStart + 2; throw new Error(msg); } else { + let startTime = Date.now(); try { await CaliperUtils.execAsync(`cd ${this.workspace}; ${command}`); } catch (err) { @@ -71,6 +71,9 @@ class CaliperEngine { logger.error(msg); this.returnCode = errorStatusStart + 3; throw new Error(msg); + } finally { + let endTime = Date.now(); + logger.info(`Executed ${commandName} command in ${(endTime - startTime)/1000.0} seconds`); } } } else { @@ -85,34 +88,35 @@ class CaliperEngine { async run() { // Retrieve flow conditioning options const flowOpts = CaliperUtils.getFlowOptions(); - let configObject = CaliperUtils.parseYaml(this.absConfigFile); - let networkObject = CaliperUtils.parseYaml(this.absNetworkFile); - // Validate configObject (benchmark configuration file) - BenchValidator.validateObject(configObject); + BenchValidator.validateObject(this.benchmarkConfig); - logger.info('####### Caliper Test #######'); - const adminClient = new Blockchain(this.admin); + logger.info('Starting benchmark flow'); + const blockchainWrapper = new Blockchain(this.blockchainAdapter); try { // Conditional running of 'start' commands if (!flowOpts.performStart) { logger.info('Skipping start commands due to benchmark flow conditioning'); } else { - await this._executeCommand(networkObject, 'start', 0); + await this._executeCommand('start', 0); } // Conditional network initialization if (!flowOpts.performInit) { logger.info('Skipping initialization phase due to benchmark flow conditioning'); } else { + let initStartTime = Date.now(); try { - await adminClient.init(); + await blockchainWrapper.init(); } catch (err) { let msg = `Error while performing "init" step: ${err.message}`; logger.error(msg); this.returnCode = 4; throw new Error(msg); + } finally { + let initEndTime = Date.now(); + logger.info(`Executed "init" step in ${(initEndTime - initStartTime)/1000.0} seconds`); } } @@ -120,13 +124,17 @@ class CaliperEngine { if (!flowOpts.performInstall) { logger.info('Skipping install smart contract phase due to benchmark flow conditioning'); } else { + let installStartTime = Date.now(); try { - await adminClient.installSmartContract(); + await blockchainWrapper.installSmartContract(); } catch (err) { let msg = `Error while performing "install" step: ${err.message}`; logger.error(msg); this.returnCode = 5; throw new Error(msg); + } finally { + let installEndTime = Date.now(); + logger.info(`Executed "install" step in ${(installEndTime - installStartTime)/1000.0} seconds`); } } @@ -134,11 +142,11 @@ class CaliperEngine { if (!flowOpts.performTest) { logger.info('Skipping benchmark test phase due to benchmark flow conditioning'); } else { - let numberSet = configObject.tests && configObject.tests.clients && configObject.tests.clients.number; - let numberOfClients = numberSet ? configObject.tests.clients.number : 1; - let clientArgs = await adminClient.prepareClients(numberOfClients); + let numberSet = this.benchmarkConfig.tests && this.benchmarkConfig.tests.clients && this.benchmarkConfig.tests.clients.number; + let numberOfClients = numberSet ? this.benchmarkConfig.tests.clients.number : 1; + let workerArguments = await blockchainWrapper.prepareClients(numberOfClients); - const roundOrchestrator = new RoundOrchestrator(clientArgs, this.absConfigFile, this.absNetworkFile, this.clientFactory, this.workspace); + const roundOrchestrator = new RoundOrchestrator(this.benchmarkConfig, this.networkConfig, this.workerFactory, workerArguments); await roundOrchestrator.run(); } } catch (err) { @@ -155,7 +163,7 @@ class CaliperEngine { logger.info('Skipping end command due to benchmark flow conditioning'); } else { try { - await this._executeCommand(networkObject, 'end', 6); + await this._executeCommand('end', 6); } catch (err) { // the error was already handled/logged, so ignore it } diff --git a/packages/caliper-core/lib/master/client/client-orchestrator.js b/packages/caliper-core/lib/master/client/client-orchestrator.js index 484aa29f1b..db2648d6ca 100644 --- a/packages/caliper-core/lib/master/client/client-orchestrator.js +++ b/packages/caliper-core/lib/master/client/client-orchestrator.js @@ -25,11 +25,14 @@ const logger = util.getLogger('client.js'); class ClientOrchestrator { /** * Constructor - * @param {String} config path of the configuration file + * @param {object} benchmarkConfig The benchmark configuration object. + * @param {object} workerFactory The factory for the worker processes. + * @param {object[]} workerArguments Array of arbitrary arguments to pass to the worker processes. */ - constructor(config) { - let conf = util.parseYaml(config); - this.config = conf.test.clients; + constructor(benchmarkConfig, workerFactory, workerArguments) { + this.config = benchmarkConfig.test.clients; + this.workerFactory = workerFactory; + this.workerArguments = workerArguments; if(this.config.hasOwnProperty('number')) { this.number = this.config.number; @@ -54,17 +57,15 @@ class ClientOrchestrator { * config: path of the blockchain config file // TODO: how to deal with the local config file when transfer it to a remote client (via zookeeper), as well as any local materials like crypto keys?? * }; * @param {JSON} test test specification - * @param {Array} clientArgs each element of the array contains arguments that should be passed to corresponding test client - * @param {Object} clientFactory a factory used to spawn test clients * @returns {Object[]} the test results array * @async */ - async startTest(test, clientArgs, clientFactory) { + async startTest(test) { this.updates.data = []; this.updates.id++; const results = []; - await this._startTest(this.number, test, clientArgs, this.updates.data, results, clientFactory); + await this._startTest(this.number, test, this.updates.data, results); const testOutput = this.formatResults(results); return testOutput; } @@ -73,13 +74,11 @@ class ClientOrchestrator { * Start a test * @param {Number} number test clients' count * @param {JSON} test test specification - * @param {Array} clientArgs each element contains specific arguments for a client * @param {Array} updates array to save txUpdate results * @param {Array} results array to save the test results - * @param {Object} clientFactory a factory to spawn test clients * @async */ - async _startTest(number, test, clientArgs, updates, results, clientFactory) { + async _startTest(number, test, updates, results) { // Conditionally launch clients on the test round. Subsequent tests should re-use existing clients. if (Object.keys(this.processes).length === 0) { @@ -87,7 +86,7 @@ class ClientOrchestrator { const readyPromises = []; this.processes = {}; for (let i = 0 ; i < number ; i++) { - this.launchClient(updates, results, clientFactory, readyPromises); + this.launchClient(updates, results, readyPromises); } // wait for all clients to have initialized logger.info(`Waiting for ${readyPromises.length} clients to be ready... `); @@ -134,7 +133,7 @@ class ClientOrchestrator { promises.push(p); client.results = results; client.updates = updates; - test.clientArgs = clientArgs[idx]; + test.clientArgs = this.workerArguments[idx]; test.clientIdx = idx; test.totalClients = number; @@ -227,11 +226,10 @@ class ClientOrchestrator { * Launch a client process to do the test * @param {Array} updates array to save txUpdate results * @param {Array} results array to save the test results - * @param {Object} clientFactory a factory to spawn clients * @param {Array} readyPromises array to hold ready promises */ - launchClient(updates, results, clientFactory, readyPromises) { - let client = clientFactory.spawnWorker(); + launchClient(updates, results, readyPromises) { + let client = this.workerFactory.spawnWorker(); let pid = client.pid.toString(); logger.info('Launching client with PID ', pid); diff --git a/packages/caliper-core/lib/master/monitor/monitor-orchestrator.js b/packages/caliper-core/lib/master/monitor/monitor-orchestrator.js index ca60cb1e45..203db40c5a 100644 --- a/packages/caliper-core/lib/master/monitor/monitor-orchestrator.js +++ b/packages/caliper-core/lib/master/monitor/monitor-orchestrator.js @@ -33,32 +33,32 @@ const VALID_MONITORS = [NONE, DOCKER, PROCESS, PROMETHEUS]; class MonitorOrchestrator { /** * Constructor - * @param {String} configPath path of the configuration file + * @param {object} benchmarkConfig The benchmark configuration object. */ - constructor(configPath) { - this.config = Util.parseYaml(configPath); + constructor(benchmarkConfig) { this.started = false; this.monitors = new Map(); // Parse the config and retrieve the monitor types - const m = this.config.monitor; - if(typeof m === 'undefined') { + const monitorConfig = benchmarkConfig.monitor; + if(typeof monitorConfig === 'undefined') { logger.info('No monitor specified, will default to "none"'); return; } - if(typeof m.type === 'undefined') { + if(typeof monitorConfig.type === 'undefined') { throw new Error('Failed to find monitor types in config file'); } - const monitorTypes = Array.isArray(m.type) ? m.type : [m.type]; + let monitorTypes = Array.isArray(monitorConfig.type) ? monitorConfig.type : [monitorConfig.type]; + monitorTypes = Array.from(new Set(monitorTypes)); // remove duplicates for (let type of monitorTypes) { let monitor = null; if(type === DOCKER) { - monitor = new DockerMonitor(m.docker, m.interval); + monitor = new DockerMonitor(monitorConfig.docker, monitorConfig.interval); } else if(type === PROCESS) { - monitor = new ProcessMonitor(m.process, m.interval); + monitor = new ProcessMonitor(monitorConfig.process, monitorConfig.interval); } else if(type === PROMETHEUS) { - monitor = new PrometheusMonitor(m.prometheus); + monitor = new PrometheusMonitor(monitorConfig.prometheus, monitorConfig.interval); } else if(type === NONE) { continue; } else { diff --git a/packages/caliper-core/lib/master/monitor/monitor-prometheus.js b/packages/caliper-core/lib/master/monitor/monitor-prometheus.js index 421b315b2b..416a5aee4c 100644 --- a/packages/caliper-core/lib/master/monitor/monitor-prometheus.js +++ b/packages/caliper-core/lib/master/monitor/monitor-prometheus.js @@ -34,7 +34,7 @@ class PrometheusMonitor extends MonitorInterface { */ constructor(monitorConfig, interval) { super(monitorConfig, interval); - this.prometheusPushClient = new PrometheusPushClient(monitorConfig.pushUrl); + this.prometheusPushClient = new PrometheusPushClient(monitorConfig.push_url); this.prometheusQueryClient = new PrometheusQueryClient(monitorConfig.url); // User defined options for monitoring if (monitorConfig.hasOwnProperty('metrics')) { diff --git a/packages/caliper-core/lib/master/report/report.js b/packages/caliper-core/lib/master/report/report.js index e068bbd86d..0d824fdb94 100644 --- a/packages/caliper-core/lib/master/report/report.js +++ b/packages/caliper-core/lib/master/report/report.js @@ -30,58 +30,60 @@ class Report { /** * Constructor for the Report object * @param {MonitorOrchestrator} monitorOrchestrator the test monitor + * @param {object} benchmarkConfig The benchmark configuration object. + * @param {object} networkConfig The network configuration object. */ - constructor(monitorOrchestrator) { + constructor(monitorOrchestrator, benchmarkConfig, networkConfig) { + this.benchmarkConfig = benchmarkConfig; + this.networkConfig = networkConfig; this.monitorOrchestrator = monitorOrchestrator; + this.reportBuilder = new ReportBuilder(); this.resultsByRound = []; this.queryClient = (monitorOrchestrator && monitorOrchestrator.hasMonitor('prometheus')) ? monitorOrchestrator.getMonitor('prometheus').getQueryClient() : null; } /** - * Generate mustache template for test report - * @param {String} absConfigFile the config file used by this flow - * @param {String} absNetworkFile the network config file used by this flow - * @param {String} blockchainType the blockchain target type + * Generate mustache template for test report. */ - createReport(absConfigFile, absNetworkFile, blockchainType) { - let config = CaliperUtils.parseYaml(absConfigFile); - this.reportBuilder.addMetadata('DLT', blockchainType); + createReport() { + let test = this.benchmarkConfig.test; + this.reportBuilder.addMetadata('DLT', this.networkConfig.caliper.blockchain); try{ - this.reportBuilder.addMetadata('Benchmark', config.test.name); + this.reportBuilder.addMetadata('Benchmark', test.name); } catch(err) { this.reportBuilder.addMetadata('Benchmark', ' '); } try { - this.reportBuilder.addMetadata('Description', config.test.description); + this.reportBuilder.addMetadata('Description', test.description); } catch(err) { this.reportBuilder.addMetadata('Description', ' '); } try{ let r = 0; - for(let i = 0 ; i < config.test.rounds.length ; i++) { - if(config.test.rounds[i].hasOwnProperty('txNumber')) { - r += config.test.rounds[i].txNumber.length; - } else if (config.test.rounds[i].hasOwnProperty('txDuration')) { - r += config.test.rounds[i].txDuration.length; + let rounds = test.rounds; + for(let i = 0 ; i < rounds.length ; i++) { + if(rounds[i].hasOwnProperty('txNumber')) { + r += rounds[i].txNumber.length; + } else if (rounds[i].hasOwnProperty('txDuration')) { + r += rounds[i].txDuration.length; } } this.reportBuilder.addMetadata('Test Rounds', r); - this.reportBuilder.setBenchmarkInfo(JSON.stringify(config.test, null, 2)); + this.reportBuilder.setBenchmarkInfo(JSON.stringify(test, null, 2)); } catch(err) { this.reportBuilder.addMetadata('Test Rounds', ' '); } - let sut = CaliperUtils.parseYaml(absNetworkFile); - if(sut.hasOwnProperty('info')) { - for(let key in sut.info) { - this.reportBuilder.addSUTInfo(key, sut.info[key]); + if(this.networkConfig.hasOwnProperty('info')) { + for(let key in this.networkConfig.info) { + this.reportBuilder.addSUTInfo(key, this.networkConfig.info[key]); } } - this.reportBuilder.addLabelDescriptionMap(config.test.rounds); + this.reportBuilder.addLabelDescriptionMap(test.rounds); } /** diff --git a/packages/caliper-core/lib/master/test-observers/local-observer.js b/packages/caliper-core/lib/master/test-observers/local-observer.js index a5673dd7f5..f6503ac735 100644 --- a/packages/caliper-core/lib/master/test-observers/local-observer.js +++ b/packages/caliper-core/lib/master/test-observers/local-observer.js @@ -26,13 +26,13 @@ class LocalObserver extends TestObserverInterface { /** * Constructor - * @param {String} configPath path of the configuration file + * @param {object} benchmarkConfig The benchmark configuration object. */ - constructor(configPath) { - super(configPath); + constructor(benchmarkConfig) { + super(benchmarkConfig); // set the observer interval - const interval = (this.config.observer && this.config.observer.interval) ? this.config.observer.interval : 1; + const interval = (this.benchmarkConfig.observer && this.benchmarkConfig.observer.interval) ? this.benchmarkConfig.observer.interval : 1; this.observeInterval = interval * 1000; Logger.info(`Observer interval set to ${interval} seconds`); this.observeIntervalObject = null; @@ -254,12 +254,12 @@ class LocalObserver extends TestObserverInterface { } /** - * Creates a new rate controller instance. - * @param {String} absConfigFile The absolute path to the benchmark config file - * @return {ObserverInterface} The rate controller instance. + * Creates a new LocalObserver instance. + * @param {object} benchmarkConfig The benchmark configuration object. + * @return {TestObserverInterface} The LocalObserver instance. */ -function createTestObserver(absConfigFile) { - return new LocalObserver(absConfigFile); +function createTestObserver(benchmarkConfig) { + return new LocalObserver(benchmarkConfig); } module.exports.createTestObserver = createTestObserver; diff --git a/packages/caliper-core/lib/master/test-observers/null-observer.js b/packages/caliper-core/lib/master/test-observers/null-observer.js index 5c79efdb6d..68c06cacb4 100644 --- a/packages/caliper-core/lib/master/test-observers/null-observer.js +++ b/packages/caliper-core/lib/master/test-observers/null-observer.js @@ -16,8 +16,7 @@ 'use strict'; const TestObserverInterface = require('./observer-interface'); -const Utils = require('../../common/utils/caliper-utils'); -const Logger = Utils.getLogger('null-observer'); +const Logger = require('../../common/utils/caliper-utils').getLogger('null-observer'); /** * NullObserver class used to omit test statistics observation @@ -26,11 +25,11 @@ class NullObserver extends TestObserverInterface { /** * Constructor - * @param {String} configPath path of the configuration file + * @param {object} benchmarkConfig The benchmark configuration object. */ - constructor(configPath) { - super(configPath); - Logger.info('Configured observer'); + constructor(benchmarkConfig) { + super(benchmarkConfig); + Logger.info('Configured "null" observer'); } /** @@ -75,12 +74,12 @@ class NullObserver extends TestObserverInterface { } /** - * Creates a new rate controller instance. - * @param {String} absConfigFile The absolute path to the benchmark config file - * @return {ObserverInterface} The rate controller instance. + * Creates a new NullObserver instance. + * @param {object} benchmarkConfig The benchmark configuration object. + * @return {TestObserverInterface} The NullObserver instance. */ -function createTestObserver(absConfigFile) { - return new NullObserver(absConfigFile); +function createTestObserver(benchmarkConfig) { + return new NullObserver(benchmarkConfig); } module.exports.createTestObserver = createTestObserver; diff --git a/packages/caliper-core/lib/master/test-observers/observer-interface.js b/packages/caliper-core/lib/master/test-observers/observer-interface.js index bffdf516a4..cfb9b08df2 100644 --- a/packages/caliper-core/lib/master/test-observers/observer-interface.js +++ b/packages/caliper-core/lib/master/test-observers/observer-interface.js @@ -14,20 +14,31 @@ 'use strict'; -const Util = require('../../common/utils/caliper-utils'); +const logger = require('../../common/utils/caliper-utils').getLogger('observer-base'); /** * Interface of test observer. Test observer implementations must follow a naming convention that is -observer.js so - * that they may be dynamically loaded within Caliper flow + * that they may be dynamically loaded in the RoundOrchestrator */ class TestObserverInterface { /** * Constructor - * @param {string} configPath the config file path + * @param {object} benchmarkConfig The benchmark configuration object. */ - constructor(configPath) { - this.config = Util.parseYaml(configPath); + constructor(benchmarkConfig) { + this.benchmarkConfig = benchmarkConfig; + } + + /** + * Logs and throws a "not implemented" error for the given function. + * @param {string} functionName The name of the function. + * @private + */ + _throwNotImplementedError(functionName) { + let msg = `The function "${functionName}" is not implemented for this test observer`; + logger.error(msg); + throw new Error(msg); } /** @@ -35,7 +46,7 @@ class TestObserverInterface { * @async */ async update() { - throw new Error('update is not implemented for this test observer'); + this._throwNotImplementedError('update'); } /** @@ -43,7 +54,7 @@ class TestObserverInterface { * @param {ClientOrchestrator} clientOrchestrator the client orchestrator */ startWatch(clientOrchestrator) { - throw new Error('startWatch is not implemented for this test observer'); + this._throwNotImplementedError('startWatch'); } /** @@ -51,7 +62,7 @@ class TestObserverInterface { * @async */ async stopWatch() { - throw new Error('stopWatch is not implemented for this test observer'); + this._throwNotImplementedError('stopWatch'); } /** @@ -59,7 +70,7 @@ class TestObserverInterface { * @param {String} name the benchmark name */ setBenchmark(name) { - throw new Error('setBenchmark is not implemented for this test observer'); + this._throwNotImplementedError('setBenchmark'); } /** @@ -67,7 +78,7 @@ class TestObserverInterface { * @param{*} roundIdx the round index */ setRound(roundIdx) { - throw new Error('setRound is not implemented for this test observer'); + this._throwNotImplementedError('setRound'); } } diff --git a/packages/caliper-core/lib/master/test-observers/prometheus-observer.js b/packages/caliper-core/lib/master/test-observers/prometheus-observer.js index 9e394b4738..fe66ce58e0 100644 --- a/packages/caliper-core/lib/master/test-observers/prometheus-observer.js +++ b/packages/caliper-core/lib/master/test-observers/prometheus-observer.js @@ -28,17 +28,17 @@ class PrometheusObserver extends TestObserverInterface { /** * Constructor - * @param {String} configPath path of the configuration file + * @param {object} benchmarkConfig The benchmark configuration object. */ - constructor(configPath) { - super(configPath); + constructor(benchmarkConfig) { + super(benchmarkConfig); // determine interval - const interval = (this.config.observer && this.config.observer.interval) ? this.config.observer.interval : 1; + const interval = (this.benchmarkConfig.observer && this.benchmarkConfig.observer.interval) ? this.benchmarkConfig.observer.interval : 1; this.observeInterval = interval * 1000; // Define the query client - const queryUrl = this.config.monitor.prometheus.url; + const queryUrl = this.benchmarkConfig.monitor.prometheus.url; this.queryClient = new PrometheusQueryClient(queryUrl); Logger.info(`Configured observer to query URL ${queryUrl} every ${interval} seconds`); } @@ -115,12 +115,12 @@ class PrometheusObserver extends TestObserverInterface { } /** - * Creates a new rate controller instance. - * @param {String} absConfigFile The absolute path to the benchmark config file - * @return {ObserverInterface} The rate controller instance. + * Creates a new PrometheusObserver instance. + * @param {object} benchmarkConfig The benchmark configuration object. + * @return {TestObserverInterface} The PrometheusObserver instance. */ -function createTestObserver(absConfigFile) { - return new PrometheusObserver(absConfigFile); +function createTestObserver(benchmarkConfig) { + return new PrometheusObserver(benchmarkConfig); } module.exports.createTestObserver = createTestObserver; diff --git a/packages/caliper-core/lib/master/test-observers/test-observer.js b/packages/caliper-core/lib/master/test-observers/test-observer.js index 0d6be30487..f58f6cbbf0 100644 --- a/packages/caliper-core/lib/master/test-observers/test-observer.js +++ b/packages/caliper-core/lib/master/test-observers/test-observer.js @@ -26,22 +26,24 @@ const TestObserver = class { /** * Instantiates the proxy test observer and creates the configured observer behind it. - * @param {String} observerType The observer to use. - * @param {String} absConfigFile The absolute path to the benchmark config file + * @param {object} benchmarkConfig The benchmark configuration object. */ - constructor(observerType, absConfigFile) { - Logger.debug(`Creating test observer of type ${observerType}`); + constructor(benchmarkConfig) { + // Test observer is dynamically loaded, but defaults to none + const observerType = (benchmarkConfig.observer && benchmarkConfig.observer.type) ? benchmarkConfig.observer.type : 'none'; + + Logger.debug(`Creating test observer of type "${observerType}"`); // resolve the type to a module path let modulePath = builtInObservers.has(observerType) - ? builtInObservers.get(observerType) : CaliperUtils.resolvePath(observerType); + ? builtInObservers.get(observerType) : CaliperUtils.resolvePath(observerType); // TODO: what if it's an external module name? let factoryFunction = require(modulePath).createTestObserver; if (!factoryFunction) { throw new Error(`${observerType} does not export the mandatory factory function 'createTestObserver'`); } - this.observer = factoryFunction(absConfigFile); + this.observer = factoryFunction(benchmarkConfig); } /** diff --git a/packages/caliper-core/lib/master/test-runners/round-orchestrator.js b/packages/caliper-core/lib/master/test-runners/round-orchestrator.js index c2f8e23a37..fc258a7ef2 100644 --- a/packages/caliper-core/lib/master/test-runners/round-orchestrator.js +++ b/packages/caliper-core/lib/master/test-runners/round-orchestrator.js @@ -20,36 +20,31 @@ const MonitorOrchestrator = require('../monitor/monitor-orchestrator'); const Report = require('../report/report'); const TestObserver = require('../test-observers/test-observer'); const CaliperUtils = require('../../common/utils/caliper-utils'); +const ConfigUtils = require('../../common/config/config-util'); const logger = CaliperUtils.getLogger('round-orchestrator'); /** - * Default testing class fot Caliper + * Schedules and drives the configured benchmark rounds. */ class RoundOrchestrator { /** - * load client(s) to do performance tests - * @param {Array} clientArgs arguments for clients - * @param {String} benchmarkConfigFile the bench config file path - * @param {String} absNetworkFile the network config file patch - * @param {Object} clientFactory factory used to spawn test clients - * @param {String} networkRoot the root location + * Initialize the RoundOrchestrator instance. + * @param {object} benchmarkConfig The benchmark configuration object. + * @param {object} networkConfig The network configuration object. + * @param {object} workerFactory The factory for worker processes. + * @param {object[]} workerArguments List of arbitrary arguments to pass for each worker processes. */ - constructor(clientArgs, benchmarkConfigFile, absNetworkFile, clientFactory, networkRoot) { - this.networkConfig = CaliperUtils.parseYaml(absNetworkFile); - this.benchmarkConfig = CaliperUtils.parseYaml(benchmarkConfigFile); - this.clientArgs = clientArgs; - this.benchmarkConfigFile = benchmarkConfigFile; - this.absNetworkFile = absNetworkFile; - this.clientFactory = clientFactory; - this.networkRoot = networkRoot; - this.clientOrchestrator = new ClientOrchestrator(this.benchmarkConfigFile); - this.monitorOrchestrator = new MonitorOrchestrator(this.benchmarkConfigFile); - this.report = new Report(this.monitorOrchestrator); - - // Test observer is dynamically loaded, but defaults to none - const observerType = (this.benchmarkConfig.observer && this.benchmarkConfig.observer.type) ? this.benchmarkConfig.observer.type : 'none'; - this.testObserver = new TestObserver(observerType, this.benchmarkConfigFile); + constructor(benchmarkConfig, networkConfig, workerFactory, workerArguments) { + this.networkConfig = networkConfig; + this.benchmarkConfig = benchmarkConfig; + + this.workspace = ConfigUtils.get(ConfigUtils.keys.Workspace); // TODO: remove this + + this.clientOrchestrator = new ClientOrchestrator(this.benchmarkConfig, workerFactory, workerArguments); + this.monitorOrchestrator = new MonitorOrchestrator(this.benchmarkConfig); + this.report = new Report(this.monitorOrchestrator, this.benchmarkConfig, this.networkConfig); + this.testObserver = new TestObserver(this.benchmarkConfig); } /** @@ -131,15 +126,13 @@ class RoundOrchestrator { * Validates and schedules the defined rounds. */ async run() { - let benchmarkConfig = CaliperUtils.parseYaml(CaliperUtils.resolvePath(this.benchmarkConfigFile, this.networkRoot)); - - if (!(benchmarkConfig.test && benchmarkConfig.test.rounds)) { + if (!(this.benchmarkConfig.test && this.benchmarkConfig.test.rounds)) { let msg = 'Benchmark configuration file is missing the "test.rounds" attribute'; logger.error(msg); throw new Error(msg); } - let rounds = benchmarkConfig.test.rounds; + let rounds = this.benchmarkConfig.test.rounds; if (!Array.isArray(rounds)) { let msg = 'Benchmark configuration attribute "test.rounds" must be an array'; logger.error(msg); @@ -149,8 +142,6 @@ class RoundOrchestrator { // validate each round before starting any rounds.forEach((round, index) => RoundOrchestrator._validateRoundConfig(round, index)); - let networkConfigPath = CaliperUtils.resolvePath(this.absNetworkFile, this.networkRoot); - // create messages for clients from each round config let roundConfigs = rounds.map((round, index) => { let config = { @@ -160,8 +151,8 @@ class RoundOrchestrator { trim: round.trim || 0, args: round.arguments, cb: round.callback, - config: networkConfigPath, - root: this.networkRoot, + config: 'TODO', // TODO: remove this + root: this.workspace, // TODO: remove this testRound: index, pushUrl: this.monitorOrchestrator.hasMonitor('prometheus') ? this.monitorOrchestrator.getMonitor('prometheus').getPushGatewayURL() : null }; @@ -178,16 +169,18 @@ class RoundOrchestrator { let success = 0; let failed = 0; - this.report.createReport(this.benchmarkConfigFile, this.absNetworkFile, this.networkConfig.caliper.blockchain); + this.report.createReport(); // Start all the monitors try { await this.monitorOrchestrator.startAllMonitors(); - logger.info('Started monitors successfully'); + logger.info('Monitors successfully started'); } catch (err) { logger.error(`Could not start monitors: ${err.stack || err}`); } + let benchStartTime = Date.now(); + for (const [index, roundConfig] of roundConfigs.entries()) { logger.info(`Started round ${index + 1} (${roundConfig.label})`); this.testObserver.setBenchmark(roundConfig.label); @@ -195,7 +188,7 @@ class RoundOrchestrator { try { this.testObserver.startWatch(this.clientOrchestrator); - const {results, start, end} = await this.clientOrchestrator.startTest(roundConfig, this.clientArgs, this.clientFactory); + const {results, start, end} = await this.clientOrchestrator.startTest(roundConfig); await this.testObserver.stopWatch(); // Build the report @@ -226,6 +219,8 @@ class RoundOrchestrator { } } + let benchEndTime = Date.now(); + // clean up, with "silent" failure handling try { this.report.printResultsByRound(); @@ -246,7 +241,7 @@ class RoundOrchestrator { logger.error(`Error while stopping clients: ${err.stack || err}`); } - logger.info(`Benchmark finished. Total rounds: ${success + failed}. Successful rounds: ${success}. Failed rounds: ${failed}.`); + logger.info(`Benchmark finished in ${(benchEndTime - benchStartTime)/1000.0} seconds. Total rounds: ${success + failed}. Successful rounds: ${success}. Failed rounds: ${failed}.`); } } diff --git a/packages/caliper-core/test/master/client/client-orchestrator.js b/packages/caliper-core/test/master/client/client-orchestrator.js index cd416a85ad..6b13d1472c 100644 --- a/packages/caliper-core/test/master/client/client-orchestrator.js +++ b/packages/caliper-core/test/master/client/client-orchestrator.js @@ -22,33 +22,33 @@ chai.should(); const sinon = require('sinon'); describe('client orchestrator implementation', () => { - + const benchmarkConfig = { + test: { + clients: { + number: 7 + } + } + }; + const workerArguments = [1,2,3]; + const workerFactory = {}; describe('#constructor', () => { it('should read the number of test clients if present in the config file', () => { - const FakeParseYaml = sinon.stub().returns({ test: { clients: {number: 7}}}); - ClientOrchestratorRewire.__set__('util.parseYaml', FakeParseYaml); + const myOrchestrator = new ClientOrchestratorRewire(benchmarkConfig, workerFactory, workerArguments); - const myOrchestrator = new ClientOrchestratorRewire(); myOrchestrator.number.should.equal(7); - }); it('should default to one client in the test if not specified in the config file ', () => { - const FakeParseYaml = sinon.stub().returns({ test: { clients: {notNumber: 2}}}); - ClientOrchestratorRewire.__set__('util.parseYaml', FakeParseYaml); + const myOrchestrator = new ClientOrchestratorRewire({ test: { clients: {notNumber: 2}}}, workerFactory, workerArguments); - const myOrchestrator = new ClientOrchestratorRewire(); myOrchestrator.number.should.equal(1); }); }); describe('#startTest', () => { - - const FakeParseYaml = sinon.stub().returns({ test: { clients: {number: 7}}}); - ClientOrchestratorRewire.__set__('util.parseYaml', FakeParseYaml); - const myOrchestrator = new ClientOrchestratorRewire(); + const myOrchestrator = new ClientOrchestratorRewire(benchmarkConfig, workerFactory, workerArguments); let _startTestStub; let formatResultsStub; @@ -62,10 +62,7 @@ describe('client orchestrator implementation', () => { it('should increment the updates.id variable', async () => { myOrchestrator.updates.id = 41; const testMsg = {msg: 'test msg'}; - const clientArgs = [1,2,3]; - const factory = {}; - - await myOrchestrator.startTest(testMsg, clientArgs, factory); + await myOrchestrator.startTest(testMsg); myOrchestrator.updates.id.should.equal(42); }); @@ -73,22 +70,16 @@ describe('client orchestrator implementation', () => { it('should call _startTest with known arguments', async () => { myOrchestrator.updates.id = 0; const testMsg = {msg: 'test msg'}; - const clientArgs = [1,2,3]; - const factory = {factory: 'a factory'}; - - await myOrchestrator.startTest(testMsg, clientArgs, factory); + await myOrchestrator.startTest(testMsg); sinon.assert.calledOnce(_startTestStub); - sinon.assert.calledWith(_startTestStub, 7, testMsg, clientArgs, [], [], factory); + sinon.assert.calledWith(_startTestStub, 7, testMsg, [], []); }); it('should call formatResults', async() => { myOrchestrator.updates.id = 0; const testMsg = {msg: 'test msg'}; - const clientArgs = [1,2,3]; - const factory = {}; - - await myOrchestrator.startTest(testMsg, clientArgs, factory); + await myOrchestrator.startTest(testMsg); sinon.assert.calledOnce(formatResultsStub); }); @@ -96,10 +87,8 @@ describe('client orchestrator implementation', () => { it('should return the response from formatResults', async () => { myOrchestrator.updates.id = 0; const testMsg = {msg: 'test msg'}; - const clientArgs = [1,2,3]; - const factory = {}; + const response = await myOrchestrator.startTest(testMsg); - const response = await myOrchestrator.startTest(testMsg, clientArgs, factory); response.should.equal('formatted'); }); @@ -107,10 +96,7 @@ describe('client orchestrator implementation', () => { describe('#getUpdates', () => { - - const FakeParseYaml = sinon.stub().returns({ test: { clients: {number: 7}}}); - ClientOrchestratorRewire.__set__('util.parseYaml', FakeParseYaml); - const myOrchestrator = new ClientOrchestratorRewire(); + const myOrchestrator = new ClientOrchestratorRewire(benchmarkConfig, workerFactory, workerArguments); it('should return the updates', () => { const checkVal = 'this is my update'; @@ -123,10 +109,7 @@ describe('client orchestrator implementation', () => { }); describe('#formatResults', () => { - - const FakeParseYaml = sinon.stub().returns({ test: { clients: {number: 7}}}); - ClientOrchestratorRewire.__set__('util.parseYaml', FakeParseYaml); - const myOrchestrator = new ClientOrchestratorRewire(); + const myOrchestrator = new ClientOrchestratorRewire(benchmarkConfig, workerFactory, workerArguments); it('should group all client results into an array under a results label', () => { const result0 = {results: [1] , start: new Date(2018, 11, 24, 10, 33), end: new Date(2018, 11, 24, 11, 33)};