From b82a3e40392d14dc44c4e49701c497d0ced35d55 Mon Sep 17 00:00:00 2001 From: Attila Klenik Date: Tue, 5 Nov 2019 19:55:42 -0100 Subject: [PATCH] Flattened round processing Signed-off-by: Attila Klenik --- .travis/avoid_verdaccio.sh | 4 +- .../caliper-core/lib/master/caliper-engine.js | 15 +- .../master/test-runners/round-orchestrator.js | 164 +++++++++++++++++- .../besu_tests/benchconfig.yaml | 12 +- .../besu_tests/networkconfig.json | 2 +- .../ethereum_tests/benchconfig.yaml | 12 +- .../ethereum_tests/networkconfig.json | 2 +- .../fabric_tests/phase1/benchconfig.yaml | 12 +- .../fabric_tests/phase2/benchconfig.yaml | 12 +- .../fabric_tests/phase3/benchconfig.yaml | 12 +- .../fabric_tests/phase4/benchconfig.yaml | 12 +- .../fabric_tests/phase5/benchconfig.yaml | 12 +- .../fisco-bcos_tests/benchconfig.yaml | 8 +- .../sawtooth_tests/benchconfig.yaml | 16 +- 14 files changed, 219 insertions(+), 76 deletions(-) diff --git a/.travis/avoid_verdaccio.sh b/.travis/avoid_verdaccio.sh index ed430b447a..e2ff057cdd 100755 --- a/.travis/avoid_verdaccio.sh +++ b/.travis/avoid_verdaccio.sh @@ -18,10 +18,10 @@ set -e set -o pipefail # Bootstrap the project again -#npm i && npm run repoclean -- --yes && npm run bootstrap +npm i && npm run repoclean -- --yes && npm run bootstrap # Run linting, license check and unit tests -#npm test +npm test # Call CLI directly # The CWD will be in one of the caliper-tests-integration/*_tests directories diff --git a/packages/caliper-core/lib/master/caliper-engine.js b/packages/caliper-core/lib/master/caliper-engine.js index 3056f8400e..6b415671d2 100644 --- a/packages/caliper-core/lib/master/caliper-engine.js +++ b/packages/caliper-core/lib/master/caliper-engine.js @@ -149,18 +149,11 @@ class CaliperEngine { logger.error('Could not start monitors, ' + (err.stack ? err.stack : err)); } - let testIdx = 0; let numberOfClients = await clientOrchestrator.init(); let clientArgs = await adminClient.prepareClients(numberOfClients); - const tester = new RoundOrchestrator(clientArgs, this.absNetworkFile, clientOrchestrator, this.clientFactory, this.workspace, report, testObserver, monitorOrchestrator); - const allTests = configObject.test.rounds; - for (let test of allTests) { - ++testIdx; - const response = await tester.runTestRounds(test, (testIdx === allTests.length)); - successes += response.successes; - failures += response.failures; - } + const tester = new RoundOrchestrator(clientArgs, this.absConfigFile, this.absNetworkFile, clientOrchestrator, this.clientFactory, this.workspace, report, testObserver, monitorOrchestrator); + const resultStats = await tester.run(); logger.info('---------- Finished Test ----------\n'); report.printResultsByRound(); @@ -171,7 +164,7 @@ class CaliperEngine { clientOrchestrator.stop(); // NOTE: keep the below multi-line formatting intact, otherwise the indents will interfere with the template literal - let testSummary = `# Test summary: ${successes} succeeded, ${failures} failed #`; + let testSummary = `# Test summary: ${resultStats.successes} succeeded, ${resultStats.failures} failed #`; logger.info(` ${'#'.repeat(testSummary.length)} @@ -183,7 +176,7 @@ ${'#'.repeat(testSummary.length)} // this means that we haven't handled/logged this failure yet if (this.returnCode < 0) { // log full stack - let msg = `Error while performing "test" step: ${err}`; + let msg = `Error while performing "test" step: ${err.stack || err}`; logger.error(msg); this.returnCode = 6; } diff --git a/packages/caliper-core/lib/master/test-runners/round-orchestrator.js b/packages/caliper-core/lib/master/test-runners/round-orchestrator.js index c32abb37fd..e11871431e 100644 --- a/packages/caliper-core/lib/master/test-runners/round-orchestrator.js +++ b/packages/caliper-core/lib/master/test-runners/round-orchestrator.js @@ -16,7 +16,7 @@ 'use strict'; const CaliperUtils = require('../../common/utils/caliper-utils'); -const logger = CaliperUtils.getLogger('defaultTest'); +const logger = CaliperUtils.getLogger('round-orchestrator'); const path = require('path'); @@ -28,6 +28,7 @@ class RoundOrchestrator { /** * load client(s) to do performance tests * @param {Array} clientArgs arguments for clients + * @param {String} benchmarkConfigFile the bench config file path * @param {String} absNetworkFile the network config file patch * @param {ClientOrchestrator} clientOrchestrator the client orchestrator * @param {Object} clientFactory factory used to spawn test clients @@ -36,18 +37,175 @@ class RoundOrchestrator { * @param {TestObserver} testObserver the test observer * @param {Object} monitorOrchestrator The monitor object */ - constructor(clientArgs, absNetworkFile, clientOrchestrator, clientFactory, networkRoot, report, testObserver, monitorOrchestrator) { + constructor(clientArgs, benchmarkConfigFile, absNetworkFile, clientOrchestrator, clientFactory, networkRoot, report, testObserver, monitorOrchestrator) { this.clientArgs = clientArgs; + this.benchmarkConfigFile = benchmarkConfigFile; this.absNetworkFile = absNetworkFile; this.clientFactory = clientFactory; this.clientOrchestrator = clientOrchestrator; this.networkRoot = networkRoot; this.report = report; - this.round = 0; this.testObserver = testObserver; this.monitorOrchestrator = monitorOrchestrator; } + static validateRoundConfig(round, index) { + // TODO: this should be converted to a joi validation + try { + if (!round.label) { + throw new Error('Missing "label" attribute'); + } + + if (typeof round.label !== 'string') { + throw new Error('"label" attribute must be a string'); + } + + if (round.txNumber) { + // excludes arrays + if (typeof round.txNumber !== 'number') { + throw new Error('"txNumber" attribute must be a number'); + } + } + + if (round.txDuration) { + // excludes arrays + if (typeof round.txDuration !== 'number') { + throw new Error('"txDuration" attribute must be a number'); + } + } + + if (round.txNumber && round.txDuration) { + throw new Error('The "txDuration" and "txNumber" attributes are mutually exclusive'); + } + + if (!round.txNumber && !round.txDuration) { + throw new Error('either the "txDuration" or the "txNumber" attribute must be specified'); + } + + if (!round.rateControl) { + throw new Error('Missing "rateControl" attribute'); + } + + if (typeof round.rateControl !== 'object') { + throw new Error('"rateControl" attribute must be an object'); + } + + if (Array.isArray(round.rateControl)) { + throw new Error('"rateControl" attribute must not be an array'); + } + + if (!round.callback) { + throw new Error('Missing "callback" attribute'); + } + + if (typeof round.callback !== 'string') { + throw new Error('"callback" attribute must be a string'); + } + + if (round.trim && typeof round.trim !== 'number') { + throw new Error('"trim" attribute must be a number'); + } + + // noinspection JSAnnotator + if (round.arguments && typeof round.arguments !== 'object') { + throw new Error('"arguments" attribute must be an object'); + } + + } catch (err) { + let msg = `Round ${index + 1} configuration validation error: ${err.message}`; + logger.error(msg); + throw new Error(msg); + } + } + + async run() { + let benchmarkConfig = CaliperUtils.parseYaml(CaliperUtils.resolvePath(this.benchmarkConfigFile, this.networkRoot)); + + if (benchmarkConfig.test && benchmarkConfig.test.rounds) { + let rounds = benchmarkConfig.test.rounds; + if (!Array.isArray(rounds)) { + let msg = 'Benchmark configuration attribute "test.rounds" must be an array'; + logger.error(msg); + throw new Error(msg); + } + + // validate each round before starting any + rounds.forEach((round, index) => RoundOrchestrator.validateRoundConfig(round, index)); + + let networkConfigPath = CaliperUtils.resolvePath(this.absNetworkFile, this.networkRoot); + + // create messages for clients from each round config + let roundConfigs = rounds.map((round, index) => { + let config = { + type: 'test', + label : round.label, + rateControl: round.rateControl, + trim: round.trim || 0, + args: round.arguments, + cb : round.callback, + config: networkConfigPath, + root: this.networkRoot, + testRound: index, + pushUrl: this.monitorOrchestrator.hasMonitor('prometheus') ? this.monitorOrchestrator.getMonitor('prometheus').getPushGatewayURL() : null + }; + + if (round.txNumber) { + config.numb = round.txNumber; + } else { + config.txDuration = round.txDuration; + } + + return config; + }); + + let successfulRoundsNum = 0; + let failedRoundsNum = 0; + + for (const [index, roundConfig] of roundConfigs.entries()) { + logger.info(`Started round ${index + 1} (${roundConfig.label})`); + this.testObserver.setRound(index); + + try { + this.testObserver.startWatch(this.clientOrchestrator); + const {results, start, end} = await this.clientOrchestrator.startTest(roundConfig, this.clientArgs, this.clientFactory); + await this.testObserver.stopWatch(); + + // Build the report + // - TPS + let idx; + if (this.monitorOrchestrator.hasMonitor('prometheus')) { + idx = await this.report.processPrometheusTPSResults({start, end}, roundConfig.label, index); + } else { + idx = await this.report.processLocalTPSResults(results, roundConfig.label); + } + + // - Resource utilization + await this.report.buildRoundResourceStatistics(idx, roundConfig.label); + + successfulRoundsNum++; + logger.info(`Finished round ${index + 1} (${roundConfig.label}) in ${(end - start)/1000.0} seconds`); + + // sleep some between the rounds + if(index !== roundConfigs.length - 1) { + logger.info('Waiting 5 seconds for the next round...'); + await CaliperUtils.sleep(5000); + await this.monitorOrchestrator.restartAllMonitors(); + } + } catch (err) { + failedRoundsNum++; + logger.error(`Failed round ${index + 1} (${roundConfig.label}): ${err.stack || err}`); + } + } + + return { successes: successfulRoundsNum, failures: failedRoundsNum }; + + } else { + let msg = 'Benchmark configuration file is missing the "test.rounds" attribute'; + logger.error(msg); + throw new Error(msg); + } + } + /** * Run test rounds * @param {JSON} args testing arguments diff --git a/packages/caliper-tests-integration/besu_tests/benchconfig.yaml b/packages/caliper-tests-integration/besu_tests/benchconfig.yaml index 04e3765a81..8350dc6fdb 100644 --- a/packages/caliper-tests-integration/besu_tests/benchconfig.yaml +++ b/packages/caliper-tests-integration/besu_tests/benchconfig.yaml @@ -19,18 +19,18 @@ test: number: 1 rounds: - label: open - txNumber: [100] - rateControl: [{ type: 'fixed-rate', opts: { tps: 10 } }] + txNumber: 100 + rateControl: { type: 'fixed-rate', opts: { tps: 10 } } arguments: money: 10000 callback: open.js - label: query - txNumber: [200] - rateControl: [{ type: 'fixed-feedback-rate', opts: { tps: 10, unfinished_per_client: 5 } }] + txNumber: 200 + rateControl: { type: 'fixed-feedback-rate', opts: { tps: 10, unfinished_per_client: 5 } } callback: query.js - label: transfer - txNumber: [100] - rateControl: [{ type: 'linear-rate', opts: { startingTps: 5, finishingTps: 20 } }] + txNumber: 100 + rateControl: { type: 'linear-rate', opts: { startingTps: 5, finishingTps: 20 } } arguments: money: 10000 callback: transfer.js diff --git a/packages/caliper-tests-integration/besu_tests/networkconfig.json b/packages/caliper-tests-integration/besu_tests/networkconfig.json index 96b61cd756..0dc37110f5 100644 --- a/packages/caliper-tests-integration/besu_tests/networkconfig.json +++ b/packages/caliper-tests-integration/besu_tests/networkconfig.json @@ -2,7 +2,7 @@ "caliper": { "blockchain": "ethereum", "command" : { - "start": "docker-compose -f config/docker-compose.yml up -d && sleep 10", + "start": "docker-compose -f config/docker-compose.yml up -d && sleep 30", "end" : "docker-compose -f config/docker-compose.yml down" } }, diff --git a/packages/caliper-tests-integration/ethereum_tests/benchconfig.yaml b/packages/caliper-tests-integration/ethereum_tests/benchconfig.yaml index 224303ddbc..f59586e34e 100644 --- a/packages/caliper-tests-integration/ethereum_tests/benchconfig.yaml +++ b/packages/caliper-tests-integration/ethereum_tests/benchconfig.yaml @@ -19,18 +19,18 @@ test: number: 2 rounds: - label: open - txNumber: [100] - rateControl: [{ type: 'fixed-rate', opts: { tps: 10 } }] + txNumber: 100 + rateControl: { type: 'fixed-rate', opts: { tps: 10 } } arguments: money: 10000 callback: open.js - label: query - txNumber: [200] - rateControl: [{ type: 'fixed-feedback-rate', opts: { tps: 10, unfinished_per_client: 5 } }] + txNumber: 200 + rateControl: { type: 'fixed-feedback-rate', opts: { tps: 10, unfinished_per_client: 5 } } callback: query.js - label: transfer - txNumber: [100] - rateControl: [{ type: 'linear-rate', opts: { startingTps: 5, finishingTps: 20 } }] + txNumber: 100 + rateControl: { type: 'linear-rate', opts: { startingTps: 5, finishingTps: 20 } } arguments: money: 10000 callback: transfer.js diff --git a/packages/caliper-tests-integration/ethereum_tests/networkconfig.json b/packages/caliper-tests-integration/ethereum_tests/networkconfig.json index 38b334fea0..e35cc04ce6 100644 --- a/packages/caliper-tests-integration/ethereum_tests/networkconfig.json +++ b/packages/caliper-tests-integration/ethereum_tests/networkconfig.json @@ -2,7 +2,7 @@ "caliper": { "blockchain": "ethereum", "command" : { - "start": "docker-compose -f config/docker-compose.yml up -d && sleep 10s", + "start": "docker-compose -f config/docker-compose.yml up -d && sleep 30s", "end" : "docker-compose -f config/docker-compose.yml down" } }, diff --git a/packages/caliper-tests-integration/fabric_tests/phase1/benchconfig.yaml b/packages/caliper-tests-integration/fabric_tests/phase1/benchconfig.yaml index ec2b0d2c89..1ab6e5719a 100644 --- a/packages/caliper-tests-integration/fabric_tests/phase1/benchconfig.yaml +++ b/packages/caliper-tests-integration/fabric_tests/phase1/benchconfig.yaml @@ -19,14 +19,14 @@ test: number: 2 rounds: - label: init1 - txNumber: [100] - rateControl: [{ type: 'fixed-rate', opts: { tps: 20 } }] + txNumber: 100 + rateControl: { type: 'fixed-rate', opts: { tps: 20 } } callback: ../init.js - label: init2 - txNumber: [200] - rateControl: [{ type: 'fixed-feedback-rate', opts: { tps: 20, unfinished_per_client: 5 } }] + txNumber: 200 + rateControl: { type: 'fixed-feedback-rate', opts: { tps: 20, unfinished_per_client: 5 } } callback: ../init.js - label: query - txNumber: [100] - rateControl: [{ type: 'linear-rate', opts: { startingTps: 10, finishingTps: 20 } }] + txNumber: 100 + rateControl: { type: 'linear-rate', opts: { startingTps: 10, finishingTps: 20 } } callback: ../query.js diff --git a/packages/caliper-tests-integration/fabric_tests/phase2/benchconfig.yaml b/packages/caliper-tests-integration/fabric_tests/phase2/benchconfig.yaml index ac46af132d..2b994378de 100644 --- a/packages/caliper-tests-integration/fabric_tests/phase2/benchconfig.yaml +++ b/packages/caliper-tests-integration/fabric_tests/phase2/benchconfig.yaml @@ -19,16 +19,16 @@ test: number: 2 rounds: - label: init1 - txNumber: [100] - rateControl: [{ type: 'fixed-rate', opts: { tps: 20 } }] + txNumber: 100 + rateControl: { type: 'fixed-rate', opts: { tps: 20 } } callback: ../init.js - label: init2 - txNumber: [200] - rateControl: [{ type: 'fixed-feedback-rate', opts: { tps: 20, unfinished_per_client: 5 } }] + txNumber: 200 + rateControl: { type: 'fixed-feedback-rate', opts: { tps: 20, unfinished_per_client: 5 } } callback: ../init.js - label: query - txNumber: [100] - rateControl: [{ type: 'linear-rate', opts: { startingTps: 10, finishingTps: 20 } }] + txNumber: 100 + rateControl: { type: 'linear-rate', opts: { startingTps: 10, finishingTps: 20 } } callback: ../query.js observer: interval: 1 diff --git a/packages/caliper-tests-integration/fabric_tests/phase3/benchconfig.yaml b/packages/caliper-tests-integration/fabric_tests/phase3/benchconfig.yaml index d31cd2899d..1f8b5870da 100644 --- a/packages/caliper-tests-integration/fabric_tests/phase3/benchconfig.yaml +++ b/packages/caliper-tests-integration/fabric_tests/phase3/benchconfig.yaml @@ -19,16 +19,16 @@ test: number: 2 rounds: - label: init1 - txNumber: [100] - rateControl: [{ type: 'fixed-rate', opts: { tps: 20 } }] + txNumber: 100 + rateControl: { type: 'fixed-rate', opts: { tps: 20 } } callback: ../init.js - label: init2 - txNumber: [200] - rateControl: [{ type: 'fixed-feedback-rate', opts: { tps: 20, unfinished_per_client: 5 } }] + txNumber: 200 + rateControl: { type: 'fixed-feedback-rate', opts: { tps: 20, unfinished_per_client: 5 } } callback: ../init.js - label: query - txNumber: [100] - rateControl: [{ type: 'linear-rate', opts: { startingTps: 10, finishingTps: 20 } }] + txNumber: 100 + rateControl: { type: 'linear-rate', opts: { startingTps: 10, finishingTps: 20 } } callback: ../query.js observer: interval: 1 diff --git a/packages/caliper-tests-integration/fabric_tests/phase4/benchconfig.yaml b/packages/caliper-tests-integration/fabric_tests/phase4/benchconfig.yaml index c8c65723fe..b19c38398c 100644 --- a/packages/caliper-tests-integration/fabric_tests/phase4/benchconfig.yaml +++ b/packages/caliper-tests-integration/fabric_tests/phase4/benchconfig.yaml @@ -19,16 +19,16 @@ test: number: 2 rounds: - label: init1 - txNumber: [100] - rateControl: [{ type: 'fixed-rate', opts: { tps: 20 } }] + txNumber: 100 + rateControl: { type: 'fixed-rate', opts: { tps: 20 } } callback: ../init.js - label: init2 - txNumber: [200] - rateControl: [{ type: 'fixed-feedback-rate', opts: { tps: 20, unfinished_per_client: 5 } }] + txNumber: 200 + rateControl: { type: 'fixed-feedback-rate', opts: { tps: 20, unfinished_per_client: 5 } } callback: ../init.js - label: query - txNumber: [100] - rateControl: [{ type: 'linear-rate', opts: { startingTps: 10, finishingTps: 20 } }] + txNumber: 100 + rateControl: { type: 'linear-rate', opts: { startingTps: 10, finishingTps: 20 } } callback: ../query.js observer: interval: 1 diff --git a/packages/caliper-tests-integration/fabric_tests/phase5/benchconfig.yaml b/packages/caliper-tests-integration/fabric_tests/phase5/benchconfig.yaml index ac46af132d..2b994378de 100644 --- a/packages/caliper-tests-integration/fabric_tests/phase5/benchconfig.yaml +++ b/packages/caliper-tests-integration/fabric_tests/phase5/benchconfig.yaml @@ -19,16 +19,16 @@ test: number: 2 rounds: - label: init1 - txNumber: [100] - rateControl: [{ type: 'fixed-rate', opts: { tps: 20 } }] + txNumber: 100 + rateControl: { type: 'fixed-rate', opts: { tps: 20 } } callback: ../init.js - label: init2 - txNumber: [200] - rateControl: [{ type: 'fixed-feedback-rate', opts: { tps: 20, unfinished_per_client: 5 } }] + txNumber: 200 + rateControl: { type: 'fixed-feedback-rate', opts: { tps: 20, unfinished_per_client: 5 } } callback: ../init.js - label: query - txNumber: [100] - rateControl: [{ type: 'linear-rate', opts: { startingTps: 10, finishingTps: 20 } }] + txNumber: 100 + rateControl: { type: 'linear-rate', opts: { startingTps: 10, finishingTps: 20 } } callback: ../query.js observer: interval: 1 diff --git a/packages/caliper-tests-integration/fisco-bcos_tests/benchconfig.yaml b/packages/caliper-tests-integration/fisco-bcos_tests/benchconfig.yaml index 1ccbfeee04..a41599e2ed 100644 --- a/packages/caliper-tests-integration/fisco-bcos_tests/benchconfig.yaml +++ b/packages/caliper-tests-integration/fisco-bcos_tests/benchconfig.yaml @@ -19,12 +19,12 @@ test: number: 2 rounds: - label: set - txNumber: [1000] - rateControl: [{ type: 'fixed-rate', opts: { tps: 100 } }] + txNumber: 1000 + rateControl: { type: 'fixed-rate', opts: { tps: 100 } } callback: set.js - label: get - txNumber: [1000] - rateControl: [{ type: 'linear-rate', opts: { startingTps: 100, finishingTps: 200 } }] + txNumber: 1000 + rateControl: { type: 'linear-rate', opts: { startingTps: 100, finishingTps: 200 } } callback: get.js observer: interval: 1 diff --git a/packages/caliper-tests-integration/sawtooth_tests/benchconfig.yaml b/packages/caliper-tests-integration/sawtooth_tests/benchconfig.yaml index 91bd6e2b22..b824ee4449 100644 --- a/packages/caliper-tests-integration/sawtooth_tests/benchconfig.yaml +++ b/packages/caliper-tests-integration/sawtooth_tests/benchconfig.yaml @@ -21,23 +21,15 @@ test: number: 2 rounds: - label: smallOperations - txNumber: - - 200 - rateControl: - - type: fixed-rate - opts: - tps: 20 + txNumber: 200 + rateControl: { type: 'fixed-rate', opts: { tps: 20 } } arguments: accounts: 30 txnPerBatch: 10 callback: smallbankOperations.js - label: query - txNumber: - - 100 - rateControl: - - type: fixed-rate - opts: - tps: 50 + txNumber: 100 + rateControl: { type: 'fixed-rate', opts: { tps: 50 } } callback: query.js observer: interval: 1