Flattened round processing

Signed-off-by: Attila Klenik <[email protected]>
hyperledger-caliper · Nov 5, 2019 · b82a3e4 · b82a3e4
1 parent 29977c9
commit b82a3e4
Show file tree

Hide file tree

Showing 14 changed files with 219 additions and 76 deletions.
diff --git a/.travis/avoid_verdaccio.sh b/.travis/avoid_verdaccio.sh
@@ -18,10 +18,10 @@ set -e
 set -o pipefail
 
 # Bootstrap the project again
-#npm i && npm run repoclean -- --yes && npm run bootstrap
+npm i && npm run repoclean -- --yes && npm run bootstrap
 
 # Run linting, license check and unit tests
-#npm test
+npm test
 
 # Call CLI directly
 # The CWD will be in one of the caliper-tests-integration/*_tests directories

diff --git a/packages/caliper-core/lib/master/caliper-engine.js b/packages/caliper-core/lib/master/caliper-engine.js
@@ -149,18 +149,11 @@ class CaliperEngine {
                     logger.error('Could not start monitors, ' + (err.stack ? err.stack : err));
                 }
 
-                let testIdx = 0;
                 let numberOfClients = await clientOrchestrator.init();
                 let clientArgs = await adminClient.prepareClients(numberOfClients);
 
-                const tester = new RoundOrchestrator(clientArgs, this.absNetworkFile, clientOrchestrator, this.clientFactory, this.workspace, report, testObserver, monitorOrchestrator);
-                const allTests = configObject.test.rounds;
-                for (let test of allTests) {
-                    ++testIdx;
-                    const response = await tester.runTestRounds(test, (testIdx === allTests.length));
-                    successes += response.successes;
-                    failures += response.failures;
-                }
+                const tester = new RoundOrchestrator(clientArgs, this.absConfigFile, this.absNetworkFile, clientOrchestrator, this.clientFactory, this.workspace, report, testObserver, monitorOrchestrator);
+                const resultStats = await tester.run();
 
                 logger.info('---------- Finished Test ----------\n');
                 report.printResultsByRound();
@@ -171,7 +164,7 @@ class CaliperEngine {
                 clientOrchestrator.stop();
 
                 // NOTE: keep the below multi-line formatting intact, otherwise the indents will interfere with the template literal
-                let testSummary = `# Test summary: ${successes} succeeded, ${failures} failed #`;
+                let testSummary = `# Test summary: ${resultStats.successes} succeeded, ${resultStats.failures} failed #`;
                 logger.info(`
 
 ${'#'.repeat(testSummary.length)}
@@ -183,7 +176,7 @@ ${'#'.repeat(testSummary.length)}
             // this means that we haven't handled/logged this failure yet
             if (this.returnCode < 0) {
                 // log full stack
-                let msg = `Error while performing "test" step: ${err}`;
+                let msg = `Error while performing "test" step: ${err.stack || err}`;
                 logger.error(msg);
                 this.returnCode = 6;
             }

diff --git a/packages/caliper-core/lib/master/test-runners/round-orchestrator.js b/packages/caliper-core/lib/master/test-runners/round-orchestrator.js
@@ -16,7 +16,7 @@
 'use strict';
 
 const CaliperUtils = require('../../common/utils/caliper-utils');
-const logger = CaliperUtils.getLogger('defaultTest');
+const logger = CaliperUtils.getLogger('round-orchestrator');
 
 const path = require('path');
 
@@ -28,6 +28,7 @@ class RoundOrchestrator {
     /**
      * load client(s) to do performance tests
      * @param {Array} clientArgs arguments for clients
+     * @param {String} benchmarkConfigFile the bench config file path
      * @param {String} absNetworkFile the network config file patch
      * @param {ClientOrchestrator} clientOrchestrator the client orchestrator
      * @param {Object} clientFactory factory used to spawn test clients
@@ -36,18 +37,175 @@ class RoundOrchestrator {
      * @param {TestObserver} testObserver the test observer
      * @param {Object} monitorOrchestrator The monitor object
      */
-    constructor(clientArgs, absNetworkFile, clientOrchestrator, clientFactory, networkRoot, report, testObserver, monitorOrchestrator) {
+    constructor(clientArgs, benchmarkConfigFile, absNetworkFile, clientOrchestrator, clientFactory, networkRoot, report, testObserver, monitorOrchestrator) {
         this.clientArgs = clientArgs;
+        this.benchmarkConfigFile = benchmarkConfigFile;
         this.absNetworkFile = absNetworkFile;
         this.clientFactory = clientFactory;
         this.clientOrchestrator = clientOrchestrator;
         this.networkRoot = networkRoot;
         this.report = report;
-        this.round = 0;
         this.testObserver = testObserver;
         this.monitorOrchestrator = monitorOrchestrator;
     }
 
+    static validateRoundConfig(round, index) {
+        // TODO: this should be converted to a joi validation
+        try {
+            if (!round.label) {
+                throw new Error('Missing "label" attribute');
+            }
+
+            if (typeof round.label !== 'string') {
+                throw new Error('"label" attribute must be a string');
+            }
+
+            if (round.txNumber) {
+                // excludes arrays
+                if (typeof round.txNumber !== 'number') {
+                    throw new Error('"txNumber" attribute must be a number');
+                }
+            }
+
+            if (round.txDuration) {
+                // excludes arrays
+                if (typeof round.txDuration !== 'number') {
+                    throw new Error('"txDuration" attribute must be a number');
+                }
+            }
+
+            if (round.txNumber && round.txDuration) {
+                throw new Error('The "txDuration" and "txNumber" attributes are mutually exclusive');
+            }
+
+            if (!round.txNumber && !round.txDuration) {
+                throw new Error('either the "txDuration" or the "txNumber" attribute must be specified');
+            }
+
+            if (!round.rateControl) {
+                throw new Error('Missing "rateControl" attribute');
+            }
+
+            if (typeof round.rateControl !== 'object') {
+                throw new Error('"rateControl" attribute must be an object');
+            }
+
+            if (Array.isArray(round.rateControl)) {
+                throw new Error('"rateControl" attribute must not be an array');
+            }
+
+            if (!round.callback) {
+                throw new Error('Missing "callback" attribute');
+            }
+
+            if (typeof round.callback !== 'string') {
+                throw new Error('"callback" attribute must be a string');
+            }
+
+            if (round.trim && typeof round.trim !== 'number') {
+                throw new Error('"trim" attribute must be a number');
+            }
+
+            // noinspection JSAnnotator
+            if (round.arguments && typeof round.arguments !== 'object') {
+                throw new Error('"arguments" attribute must be an object');
+            }
+
+        } catch (err) {
+            let msg = `Round ${index + 1} configuration validation error: ${err.message}`;
+            logger.error(msg);
+            throw new Error(msg);
+        }
+    }
+
+    async run() {
+        let benchmarkConfig = CaliperUtils.parseYaml(CaliperUtils.resolvePath(this.benchmarkConfigFile, this.networkRoot));
+
+        if (benchmarkConfig.test && benchmarkConfig.test.rounds) {
+            let rounds = benchmarkConfig.test.rounds;
+            if (!Array.isArray(rounds)) {
+                let msg = 'Benchmark configuration attribute "test.rounds" must be an array';
+                logger.error(msg);
+                throw new Error(msg);
+            }
+
+            // validate each round before starting any
+            rounds.forEach((round, index) => RoundOrchestrator.validateRoundConfig(round, index));
+
+            let networkConfigPath = CaliperUtils.resolvePath(this.absNetworkFile, this.networkRoot);
+
+            // create messages for clients from each round config
+            let roundConfigs = rounds.map((round, index) => {
+                let config = {
+                    type: 'test',
+                    label : round.label,
+                    rateControl: round.rateControl,
+                    trim: round.trim || 0,
+                    args: round.arguments,
+                    cb  : round.callback,
+                    config: networkConfigPath,
+                    root: this.networkRoot,
+                    testRound: index,
+                    pushUrl: this.monitorOrchestrator.hasMonitor('prometheus') ? this.monitorOrchestrator.getMonitor('prometheus').getPushGatewayURL() : null
+                };
+
+                if (round.txNumber) {
+                    config.numb = round.txNumber;
+                } else {
+                    config.txDuration = round.txDuration;
+                }
+
+                return config;
+            });
+
+            let successfulRoundsNum = 0;
+            let failedRoundsNum = 0;
+
+            for (const [index, roundConfig] of roundConfigs.entries()) {
+                logger.info(`Started round ${index + 1} (${roundConfig.label})`);
+                this.testObserver.setRound(index);
+
+                try {
+                    this.testObserver.startWatch(this.clientOrchestrator);
+                    const {results, start, end} = await this.clientOrchestrator.startTest(roundConfig, this.clientArgs, this.clientFactory);
+                    await this.testObserver.stopWatch();
+
+                    // Build the report
+                    // - TPS
+                    let idx;
+                    if (this.monitorOrchestrator.hasMonitor('prometheus')) {
+                        idx = await this.report.processPrometheusTPSResults({start, end}, roundConfig.label, index);
+                    } else {
+                        idx = await this.report.processLocalTPSResults(results, roundConfig.label);
+                    }
+
+                    // - Resource utilization
+                    await this.report.buildRoundResourceStatistics(idx, roundConfig.label);
+
+                    successfulRoundsNum++;
+                    logger.info(`Finished round ${index + 1} (${roundConfig.label}) in ${(end - start)/1000.0} seconds`);
+
+                    // sleep some between the rounds
+                    if(index !== roundConfigs.length - 1) {
+                        logger.info('Waiting 5 seconds for the next round...');
+                        await CaliperUtils.sleep(5000);
+                        await this.monitorOrchestrator.restartAllMonitors();
+                    }
+                } catch (err) {
+                    failedRoundsNum++;
+                    logger.error(`Failed round ${index + 1} (${roundConfig.label}): ${err.stack || err}`);
+                }
+            }
+
+            return { successes: successfulRoundsNum, failures: failedRoundsNum };
+
+        } else {
+            let msg = 'Benchmark configuration file is missing the "test.rounds" attribute';
+            logger.error(msg);
+            throw new Error(msg);
+        }
+    }
+
     /**
      * Run test rounds
      * @param {JSON} args testing arguments

diff --git a/packages/caliper-tests-integration/besu_tests/benchconfig.yaml b/packages/caliper-tests-integration/besu_tests/benchconfig.yaml
@@ -19,18 +19,18 @@ test:
         number: 1
     rounds:
     - label: open
-      txNumber: [100]
-      rateControl: [{ type: 'fixed-rate', opts: { tps: 10 } }]
+      txNumber: 100
+      rateControl: { type: 'fixed-rate', opts: { tps: 10 } }
       arguments:
           money: 10000
       callback: open.js
     - label: query
-      txNumber: [200]
-      rateControl: [{ type: 'fixed-feedback-rate', opts: { tps: 10, unfinished_per_client: 5 } }]
+      txNumber: 200
+      rateControl: { type: 'fixed-feedback-rate', opts: { tps: 10, unfinished_per_client: 5 } }
       callback: query.js
     - label: transfer
-      txNumber: [100]
-      rateControl: [{ type: 'linear-rate', opts: { startingTps: 5, finishingTps: 20 } }]
+      txNumber: 100
+      rateControl: { type: 'linear-rate', opts: { startingTps: 5, finishingTps: 20 } }
       arguments:
           money: 10000
       callback: transfer.js

diff --git a/packages/caliper-tests-integration/besu_tests/networkconfig.json b/packages/caliper-tests-integration/besu_tests/networkconfig.json
@@ -2,7 +2,7 @@
     "caliper": {
         "blockchain": "ethereum",
         "command" : {
-            "start": "docker-compose -f config/docker-compose.yml up -d && sleep 10",
+            "start": "docker-compose -f config/docker-compose.yml up -d && sleep 30",
             "end" : "docker-compose -f config/docker-compose.yml down"
           }
     },

diff --git a/packages/caliper-tests-integration/ethereum_tests/benchconfig.yaml b/packages/caliper-tests-integration/ethereum_tests/benchconfig.yaml
@@ -19,18 +19,18 @@ test:
         number: 2
     rounds:
     - label: open
-      txNumber: [100]
-      rateControl: [{ type: 'fixed-rate', opts: { tps: 10 } }]
+      txNumber: 100
+      rateControl: { type: 'fixed-rate', opts: { tps: 10 } }
       arguments:
           money: 10000
       callback: open.js
     - label: query
-      txNumber: [200]
-      rateControl: [{ type: 'fixed-feedback-rate', opts: { tps: 10, unfinished_per_client: 5 } }]
+      txNumber: 200
+      rateControl: { type: 'fixed-feedback-rate', opts: { tps: 10, unfinished_per_client: 5 } }
       callback: query.js
     - label: transfer
-      txNumber: [100]
-      rateControl: [{ type: 'linear-rate', opts: { startingTps: 5, finishingTps: 20 } }]
+      txNumber: 100
+      rateControl: { type: 'linear-rate', opts: { startingTps: 5, finishingTps: 20 } }
       arguments:
           money: 10000
       callback: transfer.js

diff --git a/packages/caliper-tests-integration/ethereum_tests/networkconfig.json b/packages/caliper-tests-integration/ethereum_tests/networkconfig.json
@@ -2,7 +2,7 @@
     "caliper": {
         "blockchain": "ethereum",
         "command" : {
-            "start": "docker-compose -f config/docker-compose.yml up -d && sleep 10s",
+            "start": "docker-compose -f config/docker-compose.yml up -d && sleep 30s",
             "end" : "docker-compose -f config/docker-compose.yml down"
           }
     },

diff --git a/packages/caliper-tests-integration/fabric_tests/phase1/benchconfig.yaml b/packages/caliper-tests-integration/fabric_tests/phase1/benchconfig.yaml
@@ -19,14 +19,14 @@ test:
         number: 2
     rounds:
     - label: init1
-      txNumber: [100]
-      rateControl: [{ type: 'fixed-rate', opts: { tps: 20 } }]
+      txNumber: 100
+      rateControl: { type: 'fixed-rate', opts: { tps: 20 } }
       callback: ../init.js
     - label: init2
-      txNumber: [200]
-      rateControl: [{ type: 'fixed-feedback-rate', opts: { tps: 20, unfinished_per_client: 5 } }]
+      txNumber: 200
+      rateControl: { type: 'fixed-feedback-rate', opts: { tps: 20, unfinished_per_client: 5 } }
       callback: ../init.js
     - label: query
-      txNumber: [100]
-      rateControl: [{ type: 'linear-rate', opts: { startingTps: 10, finishingTps: 20 } }]
+      txNumber: 100
+      rateControl: { type: 'linear-rate', opts: { startingTps: 10, finishingTps: 20 } }
       callback: ../query.js
diff --git a/packages/caliper-tests-integration/fabric_tests/phase2/benchconfig.yaml b/packages/caliper-tests-integration/fabric_tests/phase2/benchconfig.yaml
@@ -19,16 +19,16 @@ test:
         number: 2
     rounds:
     - label: init1
-      txNumber: [100]
-      rateControl: [{ type: 'fixed-rate', opts: { tps: 20 } }]
+      txNumber: 100
+      rateControl: { type: 'fixed-rate', opts: { tps: 20 } }
       callback: ../init.js
     - label: init2
-      txNumber: [200]
-      rateControl: [{ type: 'fixed-feedback-rate', opts: { tps: 20, unfinished_per_client: 5 } }]
+      txNumber: 200
+      rateControl: { type: 'fixed-feedback-rate', opts: { tps: 20, unfinished_per_client: 5 } }
       callback: ../init.js
     - label: query
-      txNumber: [100]
-      rateControl: [{ type: 'linear-rate', opts: { startingTps: 10, finishingTps: 20 } }]
+      txNumber: 100
+      rateControl: { type: 'linear-rate', opts: { startingTps: 10, finishingTps: 20 } }
       callback: ../query.js
 observer:
     interval: 1

diff --git a/packages/caliper-tests-integration/fabric_tests/phase3/benchconfig.yaml b/packages/caliper-tests-integration/fabric_tests/phase3/benchconfig.yaml
@@ -19,16 +19,16 @@ test:
         number: 2
     rounds:
     - label: init1
-      txNumber: [100]
-      rateControl: [{ type: 'fixed-rate', opts: { tps: 20 } }]
+      txNumber: 100
+      rateControl: { type: 'fixed-rate', opts: { tps: 20 } }
       callback: ../init.js
     - label: init2
-      txNumber: [200]
-      rateControl: [{ type: 'fixed-feedback-rate', opts: { tps: 20, unfinished_per_client: 5 } }]
+      txNumber: 200
+      rateControl: { type: 'fixed-feedback-rate', opts: { tps: 20, unfinished_per_client: 5 } }
       callback: ../init.js
     - label: query
-      txNumber: [100]
-      rateControl: [{ type: 'linear-rate', opts: { startingTps: 10, finishingTps: 20 } }]
+      txNumber: 100
+      rateControl: { type: 'linear-rate', opts: { startingTps: 10, finishingTps: 20 } }
       callback: ../query.js
 observer:
     interval: 1