From 3fcec80b362abd872760c87440e64d7d16c6d8a7 Mon Sep 17 00:00:00 2001 From: "nkl199@yahoo.co.uk" Date: Wed, 6 Nov 2019 11:33:55 +0000 Subject: [PATCH] Enable charts to be included in output report - Breaking changes in monitor configuration for docker and process monitors - Report styling change - Addition of charts to output report (opt in per monitor) - Conversion of report tables to be normalized to a single metric - Report presicion now a user config item Signed-off-by: nkl199@yahoo.co.uk --- .../caliper-core/lib/common/config/Config.js | 8 +- .../lib/common/config/default.yaml | 11 +- .../lib/master/charts/chart-builder.js | 252 +++++++++++++++ .../lib/master/monitor/monitor-docker.js | 146 +++++---- .../master/monitor/monitor-orchestrator.js | 5 +- .../lib/master/monitor/monitor-process.js | 36 ++- .../lib/master/monitor/monitor-prometheus.js | 34 +- .../lib/master/monitor/monitor-utilities.js | 60 ++++ .../lib/master/report/report-builder.js | 127 +++++--- .../caliper-core/lib/master/report/report.js | 50 ++- .../lib/master/report/template/report.html | 269 +++++++++------- .../master/test-runners/round-orchestrator.js | 7 +- packages/caliper-core/package.json | 1 + .../test/master/charts/chart-builder.js | 292 ++++++++++++++++++ .../test/master/monitor/monitor-utilities.js | 174 +++++++++++ .../fabric_tests/phase2/benchconfig.yaml | 3 +- .../fabric_tests/phase3/benchconfig.yaml | 5 +- .../fabric_tests/phase4/benchconfig.yaml | 5 + .../fabric_tests/phase5/benchconfig.yaml | 3 +- 19 files changed, 1207 insertions(+), 281 deletions(-) create mode 100644 packages/caliper-core/lib/master/charts/chart-builder.js create mode 100644 packages/caliper-core/test/master/charts/chart-builder.js create mode 100644 packages/caliper-core/test/master/monitor/monitor-utilities.js diff --git a/packages/caliper-core/lib/common/config/Config.js b/packages/caliper-core/lib/common/config/Config.js index 3bd1d7433..13a696067 100644 --- a/packages/caliper-core/lib/common/config/Config.js +++ b/packages/caliper-core/lib/common/config/Config.js @@ -29,7 +29,13 @@ const keys = { }, Report: { Path: 'caliper-report-path', - Options: 'caliper-report-options' + Options: 'caliper-report-options', + Precision: 'caliper-report-precision', + Charting: { + Hue: 'caliper-report-charting-hue', + Scheme: 'caliper-report-charting-scheme', + Transparency: 'caliper-report-charting-transparency' + } }, Workspace: 'caliper-workspace', ProjectConfig: 'caliper-projectconfig', diff --git a/packages/caliper-core/lib/common/config/default.yaml b/packages/caliper-core/lib/common/config/default.yaml index e9603c003..ee4a476ab 100644 --- a/packages/caliper-core/lib/common/config/default.yaml +++ b/packages/caliper-core/lib/common/config/default.yaml @@ -31,6 +31,13 @@ caliper: options: flag: 'w' mode: 0666 + # Precision (significant figures) for the report output + precision: 3 + # Charting options + charting: + hue: 21 + scheme: 'triade' + transparency: 0.6 # Workspace directory that contains all configuration information workspace: './' # The file path for the project-level configuration file. Can be relative to the workspace. @@ -43,10 +50,6 @@ caliper: benchconfig: # Path to the blockchain configuration file that contains information required to interact with the SUT networkconfig: - # Address of a ZooKeeper node used by distributed clients - zooaddress: - # Path to a zookeeper service yaml file - zooconfig: # Sets the frequency of the progress reports in milliseconds txupdatetime: 1000 # Configurations related to the logging mechanism diff --git a/packages/caliper-core/lib/master/charts/chart-builder.js b/packages/caliper-core/lib/master/charts/chart-builder.js new file mode 100644 index 000000000..2198dc9b3 --- /dev/null +++ b/packages/caliper-core/lib/master/charts/chart-builder.js @@ -0,0 +1,252 @@ +/* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + + +'use strict'; + +const Util = require('../../common/utils/caliper-utils'); +const Logger = Util.getLogger('chart-builder'); +const ConfigUtil = require('../../common/config/config-util'); + +const ColorScheme = require('color-scheme'); + +/** + * ChartBuilder class for generating chart information to include in a test report + */ +class ChartBuilder { + + /** + * Return chart statistics for the passed resourceStats + * @param {string} callingMonitor the calling class name of the monitor, required to create a UUID + * @param {Object} chartTypes the chart information for a monitor + * @param {string} testLabel the current test label, required to create a UUID + * @param {Map[]} resourceStatsArray the resource stats to work with + * @returns {Object[]} an array of chart stats object + */ + static retrieveChartStats(callingMonitor, chartTypes, testLabel, resourceStatsArray) { + const chartArray = []; + // Produce each chart type requested + for (const chartType of Object.keys(chartTypes)) { + const metrics = chartTypes[chartType].metrics; + const includeMetrics = ChartBuilder.retrieveIncludedMetrics(callingMonitor, chartType, metrics, resourceStatsArray[0]); + switch (chartType) { + case 'bar': { + const barCharts = ChartBuilder.barChart(callingMonitor, testLabel, includeMetrics, resourceStatsArray); + chartArray.push(...barCharts); + break; + } + case 'polar': { + const polarCharts = ChartBuilder.polarChart(callingMonitor, testLabel, includeMetrics, resourceStatsArray); + chartArray.push(...polarCharts); + break; + } + default: + Logger.error(`Unknown chart type named "${chartType}" requested`); + } + } + + return chartArray; + } + + /** + * Determine the metric names to include + * @param {string} callingMonitor calling className + * @param {string} chartType chart type + * @param {string[]} includeMetrics items to include ['all'] or ['item0', 'item1', ...] + * @param {Map} resourceStats a resource stat + * @returns {string[]} a string array of metric names + */ + static retrieveIncludedMetrics(callingMonitor, chartType, includeMetrics, resourceStats) { + // includeMetrics must exist! + if (!includeMetrics) { + Logger.error(`Required "metrics" not provided for ${chartType} chart generation for monitor ${callingMonitor}`); + return []; + } + + // Cannot have 'all' with other items + if (includeMetrics.indexOf('all') !== -1 && includeMetrics.length > 1) { + Logger.error(`Cannot list "all" option with other metrics for ${chartType} chart generation for monitor ${callingMonitor}`); + return []; + } + + const metrics = []; + for (const metric of resourceStats.keys()) { + // Do not include 'Name' + if ((metric.localeCompare('Name') === 0) || (metric.localeCompare('Type') === 0)) { + continue; + } + + // filter on all + if (includeMetrics[0].toLowerCase().localeCompare('all') === 0) { + Logger.debug(`Detected "all" option for ${chartType} chart generation for monitor ${callingMonitor}: including metric ${metric}`); + metrics.push(metric); + continue; + } + + // filter on include + for (const include of includeMetrics) { + // beware of appended units to metric + if (metric.includes(include)) { + Logger.debug(`Detected metric match on ${include} for ${chartType} chart generation for monitor ${callingMonitor}: including metric ${metric}`); + metrics.push(metric); + continue; + } + } + } + return metrics; + } + + /** + * Extract red component from hex value + * @param {hex} h the hex code + * @returns {number} the R component (0 -> 256) + */ + static hexToR(h) { + return parseInt((ChartBuilder.cutHex(h)).substring(0,2),16); + } + + /** + * Extract green component from hex value + * @param {hex} h the hex code + * @returns {number} the G component (0 -> 256) + */ + static hexToG(h) { + return parseInt((ChartBuilder.cutHex(h)).substring(2,4),16); + } + + /** + * Extract blue component from hex value + * @param {hex} h the hex code + * @returns {number} the B component (0 -> 256) + */ + static hexToB(h) { + return parseInt((ChartBuilder.cutHex(h)).substring(4,6),16); + } + + /** + * Extract red component from hex value + * @param {hex} h the hex code + * @returns {hex} the hex code without the leading # + */ + static cutHex(h) { + return (h.charAt(0)==='#') ? h.substring(1,7) : h; + } + + /** + * Use for building a bar or polar chart + * { + 'chart-id': 'UUID for chart', + 'chart-data': JSON.stringify({ + labels: [obj0, obj1, obj2, ...] + type: 'bar' | 'polar', + title: metric name, + legend: false, + datasets: [ + { + backgroundColor, + data: [obj0_data, obj1_data, obj2_data, ...], + } + ] + }) + } + * + * @param {string} callingMonitor the calling class name used to create a UUID + * @param {string} testLabel the current test label, required for creating a UUID + * @param {string[]} includeMetrics the metrics to include in the chart + * @param {Map[]} resourceStatsArray all resource stats + * @param {string} chartType the chart type (bar or polar) + * @param {boolean} legend boolean flag to include legend or not + * @returns {Object[]} an array of charting objects + */ + static _basicChart(callingMonitor, testLabel, includeMetrics, resourceStatsArray, chartType, legend){ + // Loop over desired metrics + const charts = []; + let chartIndex = 0; + for (const metricName of includeMetrics) { + + // Top Level Item + const chart = { + 'chart-id': `${callingMonitor}_${testLabel}_${chartType}${chartIndex++}`, + }; + + // Chart Data - building on names + const chartData = { + type: chartType, + title: metricName, + legend: legend, + }; + + // Pull charting options from config to retrieve charting colours + const hue = ConfigUtil.get(ConfigUtil.keys.Report.Charting.Hue, 21); + const scheme = ConfigUtil.get(ConfigUtil.keys.Report.Charting.Scheme, 'triade'); + const transparency = ConfigUtil.get(ConfigUtil.keys.Report.Charting.Transparency, 0.6); + const labels = []; + const data = []; + const backgroundColor = []; + const colourScheme = new ColorScheme; + const colours = colourScheme.from_hue(hue).scheme(scheme).colors(); + let i=0; + for (const stat of resourceStatsArray) { + // condition the retrieved value (numeric or zero) + const value = stat.get(metricName); + data.push(isNaN(value) ? 0 : value); + labels.push(stat.get('Name')); + // use rgba values so that we can include a transparency + backgroundColor.push(`rgb(${ChartBuilder.hexToR(colours[i])},${ChartBuilder.hexToG(colours[i])},${ChartBuilder.hexToB(colours[i])},${transparency})`); + i++; + // We might have more data points than available colours, so reset the index if required + if (i>=colours.length) { + i = 0; + } + } + + chartData.labels = labels; + chartData.datasets = [{backgroundColor, data}]; + chart['chart-data'] = JSON.stringify(chartData); + charts.push(chart); + } + return charts; + } + + /** + * Output an object that may be used by the template engine to render a bar chart within charting.js + * + * Bar charts assume the output of a single chart per metric, for all known names + * + * @param {string} callingMonitor the calling class, required for creating a UUID + * @param {string} testLabel the current test label, required for creating a UUID + * @param {string[]} includeMetrics metrics to be included in the output chart + * @param {Map[]} resourceStatsArray resource statistics to inspect and extract information from + * @returns {Object[]} an array of charting objects + */ + static barChart(callingMonitor, testLabel, includeMetrics, resourceStatsArray) { + return ChartBuilder._basicChart(callingMonitor, testLabel, includeMetrics, resourceStatsArray, 'horizontalBar', false); + } + + /** + * Polar charts assume the output of a single chart per metric, for all known names + * + * @param {string} callingMonitor the calling class, required for creating a UUID + * @param {string} testLabel the current test label, required for creating a UUID + * @param {string[]} includeMetrics metrics to be included in the output chart + * @param {Map[]} resourceStatsArray resource statistics to inspect and extract information from + * @returns {Object[]} an array of charting objects + */ + static polarChart(callingMonitor, testLabel, includeMetrics, resourceStatsArray) { + return ChartBuilder._basicChart(callingMonitor, testLabel, includeMetrics, resourceStatsArray, 'polarArea', true); + } + +} + +module.exports = ChartBuilder; diff --git a/packages/caliper-core/lib/master/monitor/monitor-docker.js b/packages/caliper-core/lib/master/monitor/monitor-docker.js index 702778158..dcb39d8b1 100644 --- a/packages/caliper-core/lib/master/monitor/monitor-docker.js +++ b/packages/caliper-core/lib/master/monitor/monitor-docker.js @@ -19,6 +19,7 @@ const Util = require('../../common/utils/caliper-utils'); const Logger = Util.getLogger('monitor-docker'); const MonitorInterface = require('./monitor-interface'); const MonitorUtilities = require('./monitor-utilities'); +const ChartBuilder = require('../charts/chart-builder'); const URL = require('url'); const Docker = require('dockerode'); @@ -35,10 +36,10 @@ class MonitorDocker extends MonitorInterface { */ constructor(monitorConfig, interval) { super(monitorConfig, interval); - this.containers = null; - this.isReading = false; - this.intervalObj = null; - this.stats = {'time': []}; + this.containers = null; + this.isReading = false; + this.intervalObj = null; + this.stats = { 'time': [] }; /* this.stats : used to record statistics of each container { 'time' : [] // time slot @@ -63,47 +64,47 @@ class MonitorDocker extends MonitorInterface { */ async findContainers() { this.containers = []; - let filterName = {local:[], remote:{}}; + let filterName = { local: [], remote: {} }; // Split docker items that are local or remote - if(this.monitorConfig.hasOwnProperty('name')) { - for(let key in this.monitorConfig.name) { - let name = this.monitorConfig.name[key]; - if(name.indexOf('http://') === 0) { + if (this.monitorConfig.hasOwnProperty('containers')) { + for (let key in this.monitorConfig.containers) { + let container = this.monitorConfig.containers[key]; + if (container.indexOf('http://') === 0) { // Is remote - let remote = URL.parse(name, true); - if(remote.hostname === null || remote.port === null || remote.pathname === '/') { - Logger.warn('unrecognized host, ' + name); - } else if(filterName.remote.hasOwnProperty(remote.hostname)) { + let remote = URL.parse(container, true); + if (remote.hostname === null || remote.port === null || remote.pathname === '/') { + Logger.warn('unrecognized host, ' + container); + } else if (filterName.remote.hasOwnProperty(remote.hostname)) { filterName.remote[remote.hostname].containers.push(remote.pathname); } else { - filterName.remote[remote.hostname] = {port: remote.port, containers: [remote.pathname]}; + filterName.remote[remote.hostname] = { port: remote.port, containers: [remote.pathname] }; } } else { // Is local - filterName.local.push(name); + filterName.local.push(container); } } } // Filter local containers by name - if(filterName.local.length > 0) { + if (filterName.local.length > 0) { try { const containers = await SystemInformation.dockerContainers('active'); let size = containers.length; - if(size === 0) { + if (size === 0) { Logger.error('Could not find any active local containers'); } else { - if(filterName.local.indexOf('all') !== -1) { + if (filterName.local.indexOf('all') !== -1) { // Add all containers - for(let i = 0 ; i < size ; i++){ - this.containers.push({id: containers[i].id, name: containers[i].name, remote: null}); + for (let i = 0; i < size; i++) { + this.containers.push({ id: containers[i].id, name: containers[i].name, remote: null }); this.stats[containers[i].id] = this.newContainerStat(); } } else { // Filter containers - for(let i = 0 ; i < size ; i++){ - if(filterName.local.indexOf(containers[i].name) !== -1) { - this.containers.push({id: containers[i].id, name: containers[i].name, remote: null}); + for (let i = 0; i < size; i++) { + if (filterName.local.indexOf(containers[i].name) !== -1) { + this.containers.push({ id: containers[i].id, name: containers[i].name, remote: null }); this.stats[containers[i].id] = this.newContainerStat(); } } @@ -114,7 +115,7 @@ class MonitorDocker extends MonitorInterface { } } // Filter remote containers by name - for(let h in filterName.remote) { + for (let h in filterName.remote) { try { // Instantiate for the host/port let docker = new Docker({ @@ -127,17 +128,17 @@ class MonitorDocker extends MonitorInterface { if (containers.length === 0) { Logger.error('monitor-docker: could not find remote container at ' + h); } else { - if(filterName.remote[h].containers.indexOf('/all') !== -1) { - for(let i = 0 ; i < containers.length ; i++) { + if (filterName.remote[h].containers.indexOf('/all') !== -1) { + for (let i = 0; i < containers.length; i++) { let container = docker.getContainer(containers[i].Id); - this.containers.push({id: containers[i].Id, name: h + containers[i].Names[0], remote: container}); + this.containers.push({ id: containers[i].Id, name: h + containers[i].Names[0], remote: container }); this.stats[containers[i].Id] = this.newContainerStat(); } } else { - for(let i = 0 ; i < containers.length ; i++) { - if(filterName.remote[h].containers.indexOf(containers[i].Names[0]) !== -1) { + for (let i = 0; i < containers.length; i++) { + if (filterName.remote[h].containers.indexOf(containers[i].Names[0]) !== -1) { let container = docker.getContainer(containers[i].Id); - this.containers.push({id: containers[i].Id, name: h + containers[i].Names[0], remote: container}); + this.containers.push({ id: containers[i].Id, name: h + containers[i].Names[0], remote: container }); this.stats[containers[i].Id] = this.newContainerStat(); } } @@ -155,13 +156,13 @@ class MonitorDocker extends MonitorInterface { */ newContainerStat() { return { - mem_usage: [], + mem_usage: [], mem_percent: [], cpu_percent: [], - netIO_rx: [], - netIO_tx: [], - blockIO_rx: [], - blockIO_wx: [] + netIO_rx: [], + netIO_tx: [], + blockIO_rx: [], + blockIO_wx: [] }; } @@ -177,19 +178,19 @@ class MonitorDocker extends MonitorInterface { this.isReading = true; let startPromises = []; try { - for (let i = 0 ;i < this.containers.length ; i++){ + for (let i = 0; i < this.containers.length; i++) { if (this.containers[i].remote === null) { // local startPromises.push(SystemInformation.dockerContainerStats(this.containers[i].id)); } else { // remote - startPromises.push(this.containers[i].remote.stats({stream: false})); + startPromises.push(this.containers[i].remote.stats({ stream: false })); } } - const results = await Promise.all(startPromises); - this.stats.time.push(Date.now()/1000); - for (let i = 0 ; i < results.length ; i++) { + const results = await Promise.all(startPromises); + this.stats.time.push(Date.now() / 1000); + for (let i = 0; i < results.length; i++) { let stat = results[i]; let id = stat.id; if (this.containers.length <= i) { @@ -206,7 +207,7 @@ class MonitorDocker extends MonitorInterface { let cpuDelta = stat.cpu_stats.cpu_usage.total_usage - stat.precpu_stats.cpu_usage.total_usage; let sysDelta = stat.cpu_stats.system_cpu_usage - stat.precpu_stats.system_cpu_usage; if (cpuDelta > 0 && sysDelta > 0) { - if(stat.cpu_stats.cpu_usage.hasOwnProperty('percpu_usage') && stat.cpu_stats.cpu_usage.percpu_usage !== null) { + if (stat.cpu_stats.cpu_usage.hasOwnProperty('percpu_usage') && stat.cpu_stats.cpu_usage.percpu_usage !== null) { this.stats[id].cpu_percent.push(cpuDelta / sysDelta * this.coresInUse(stat.cpu_stats) * 100.0); } else { this.stats[id].cpu_percent.push(cpuDelta / sysDelta * 100.0); @@ -243,14 +244,14 @@ class MonitorDocker extends MonitorInterface { this.stats[id].netIO_rx.push(ioRx); this.stats[id].netIO_tx.push(ioTx); let diskR = 0, diskW = 0; - if (stat.blkio_stats && stat.blkio_stats.hasOwnProperty('io_service_bytes_recursive')){ + if (stat.blkio_stats && stat.blkio_stats.hasOwnProperty('io_service_bytes_recursive')) { //Logger.debug(stat.blkio_stats.io_service_bytes_recursive); let temp = stat.blkio_stats.io_service_bytes_recursive; - for (let dIo =0; dIo detailing key/value pairs - * @return {Map[]} an array of resource maps for watched containers + * @param {string} testLabel the current test label + * @return {Object} an object containing an array of resource maps for watched containers, and a possible array of charting information * @async */ - async getStatistics() { + async getStatistics(testLabel) { try { - const watchItemStats = []; + const resourceStats = []; + // Build a statistic for each monitored container and push into watchItems array for (const container of this.containers) { if (container.hasOwnProperty('id')) { @@ -356,27 +357,40 @@ class MonitorDocker extends MonitorInterface { // Store in a Map const watchItemStat = this.getResultColumnMap(); watchItemStat.set('Name', container.name); - watchItemStat.set('Memory(max)', MonitorUtilities.byteNormalize(mem_stat.max)); - watchItemStat.set('Memory(avg)', MonitorUtilities.byteNormalize(mem_stat.avg)); + watchItemStat.set('Memory(max)', mem_stat.max); + watchItemStat.set('Memory(avg)', mem_stat.avg); watchItemStat.set('CPU% (max)', cpu_stat.max.toFixed(2)); watchItemStat.set('CPU% (avg)', cpu_stat.avg.toFixed(2)); - watchItemStat.set('Traffic In', MonitorUtilities.byteNormalize((net.in[net.in.length-1] - net.in[0]))); - watchItemStat.set('Traffic Out', MonitorUtilities.byteNormalize((net.out[net.out.length-1] - net.out[0]))); - watchItemStat.set('Disc Write', MonitorUtilities.byteNormalize((disc.write[disc.write.length-1] - disc.write[0]))); - watchItemStat.set('Disc Read', MonitorUtilities.byteNormalize((disc.read[disc.read.length-1] - disc.read[0]))); + watchItemStat.set('Traffic In', (net.in[net.in.length - 1] - net.in[0])); + watchItemStat.set('Traffic Out', (net.out[net.out.length - 1] - net.out[0])); + watchItemStat.set('Disc Write', (disc.write[disc.write.length - 1] - disc.write[0])); + watchItemStat.set('Disc Read', (disc.read[disc.read.length - 1] - disc.read[0])); // append return array - watchItemStats.push(watchItemStat); + resourceStats.push(watchItemStat); } } - return watchItemStats; - } catch(error) { + + // Normalize the resource stats to a single unit + const normalizeStats = ['Memory(max)', 'Memory(avg)', 'Traffic In', 'Traffic Out', 'Disc Write', 'Disc Read']; + for (const stat of normalizeStats) { + MonitorUtilities.normalizeStats(stat, resourceStats); + } + + // Retrieve Chart data + const chartTypes = this.monitorConfig.charting; + let chartStats = []; + if (chartTypes) { + chartStats = ChartBuilder.retrieveChartStats(this.constructor.name, chartTypes, testLabel, resourceStats); + } + + return { resourceStats, chartStats }; + } catch (error) { Logger.error('Failed to read monitoring data, ' + (error.stack ? error.stack : error)); return []; } } - /** * Get history of memory usage * @param {String} key key of the container @@ -401,7 +415,7 @@ class MonitorDocker extends MonitorInterface { * @return {Array} array of network IO usage */ getNetworkHistory(key) { - return {'in': this.stats[key].netIO_rx, 'out': this.stats[key].netIO_tx}; + return { 'in': this.stats[key].netIO_rx, 'out': this.stats[key].netIO_tx }; } /** @@ -410,7 +424,7 @@ class MonitorDocker extends MonitorInterface { * @return {Array} array of disc usage */ getDiscHistory(key) { - return {'read': this.stats[key].blockIO_rx, 'write': this.stats[key].blockIO_wx}; + return { 'read': this.stats[key].blockIO_rx, 'write': this.stats[key].blockIO_wx }; } /** diff --git a/packages/caliper-core/lib/master/monitor/monitor-orchestrator.js b/packages/caliper-core/lib/master/monitor/monitor-orchestrator.js index 203db40c5..c286f3dfd 100644 --- a/packages/caliper-core/lib/master/monitor/monitor-orchestrator.js +++ b/packages/caliper-core/lib/master/monitor/monitor-orchestrator.js @@ -150,11 +150,12 @@ class MonitorOrchestrator { /** * Get an Array of statistics maps for a named resource monitor * @param {String} type the monitor type + * @param {String} testLabel the current test label * @return {Map[]} an array of resource maps * @async */ - async getStatisticsForMonitor(type) { - return await this.monitors.get(type).getStatistics(); + async getStatisticsForMonitor(type, testLabel) { + return await this.monitors.get(type).getStatistics(testLabel); } } diff --git a/packages/caliper-core/lib/master/monitor/monitor-process.js b/packages/caliper-core/lib/master/monitor/monitor-process.js index a98fbae98..b55b75ffd 100644 --- a/packages/caliper-core/lib/master/monitor/monitor-process.js +++ b/packages/caliper-core/lib/master/monitor/monitor-process.js @@ -18,6 +18,7 @@ const MonitorInterface = require('./monitor-interface'); const MonitorUtilities = require('./monitor-utilities'); const Util = require('../../common/utils/caliper-utils'); const Logger = Util.getLogger('monitor-process'); +const ChartBuilder = require('../charts/chart-builder'); const ps = require('ps-node'); const usage = require('pidusage'); @@ -48,11 +49,11 @@ class MonitorProcess extends MonitorInterface { */ this.stats = {'time': []}; this.watchItems = []; - for(let i = 0 ; i < this.monitorConfig.length ; i++) { - if(this.monitorConfig[i].hasOwnProperty('command')) { - let id = this.getId(this.monitorConfig[i]); + for(let i = 0 ; i < this.monitorConfig.processes.length ; i++) { + if(this.monitorConfig.processes[i].hasOwnProperty('command')) { + let id = this.getId(this.monitorConfig.processes[i]); this.stats[id] = this.newStat(); - this.watchItems.push(this.monitorConfig[i]); + this.watchItems.push(this.monitorConfig.processes[i]); } } } @@ -254,26 +255,25 @@ class MonitorProcess extends MonitorInterface { * @return {Map} Map of items to build results for, with default null entries */ getResultColumnMap() { - const columns = ['Type', 'Name', 'Memory(max)', 'Memory(avg)', 'CPU% (max)', 'CPU% (avg)']; + const columns = ['Name', 'Memory(max)', 'Memory(avg)', 'CPU%(max)', 'CPU%(avg)']; const resultMap = new Map(); for (const item of columns) { resultMap.set(item, 'N/A'); } - // This is a Process Type, so set here - resultMap.set('Type', 'Process'); return resultMap; } /** * Get statistics from the monitor in the form of an Array containing Map detailing key/value pairs + * @param {string} testLabel the current test label * @return {Map[]} an array of resource maps for watched containers * @async */ - async getStatistics() { + async getStatistics(testLabel) { try { - const watchItemStats = []; + const resourceStats = []; for (const watchItem of this.watchItems) { const key = this.getId(watchItem); @@ -292,9 +292,23 @@ class MonitorProcess extends MonitorInterface { watchItemStat.set('CPU% (avg)', cpu_stat.avg.toFixed(2)); // append return array - watchItemStats.push(watchItemStat); + resourceStats.push(watchItemStat); } - return watchItemStats; + + // Normalize the resource stats to a single unit + const normalizeStats = ['Memory(max)', 'Memory(avg)', 'CPU% (max)', 'CPU% (avg)']; + for (const stat of normalizeStats) { + MonitorUtilities.normalizeStats(stat, resourceStats); + } + + // Retrieve Chart data + const chartTypes = this.monitorConfig.charting; + let chartStats = []; + if (chartTypes) { + chartStats = ChartBuilder.retrieveChartStats(this.constructor.name, chartTypes, testLabel, resourceStats); + } + + return { resourceStats, chartStats }; } catch (error) { Logger.error('Failed to read monitoring data, ' + (error.stack ? error.stack : error)); return []; diff --git a/packages/caliper-core/lib/master/monitor/monitor-prometheus.js b/packages/caliper-core/lib/master/monitor/monitor-prometheus.js index 416a5aee4..2b475fb9f 100644 --- a/packages/caliper-core/lib/master/monitor/monitor-prometheus.js +++ b/packages/caliper-core/lib/master/monitor/monitor-prometheus.js @@ -15,7 +15,8 @@ 'use strict'; const Util = require('../../common/utils/caliper-utils.js'); - +const ConfigUtil = require('../../common/config/config-util'); +const ChartBuilder = require('../charts/chart-builder'); const Logger = Util.getLogger('monitor-prometheus.js'); const MonitorInterface = require('./monitor-interface'); const PrometheusPushClient = require('../../common/prometheus/prometheus-push-client'); @@ -34,6 +35,7 @@ class PrometheusMonitor extends MonitorInterface { */ constructor(monitorConfig, interval) { super(monitorConfig, interval); + this.precision = ConfigUtil.get(ConfigUtil.keys.Report.Precision, 3); this.prometheusPushClient = new PrometheusPushClient(monitorConfig.push_url); this.prometheusQueryClient = new PrometheusQueryClient(monitorConfig.url); // User defined options for monitoring @@ -120,15 +122,16 @@ class PrometheusMonitor extends MonitorInterface { /** * Get statistics from Prometheus via queries that target the Prometheus server + * @param {string} testLabel the current test label * @returns {Map[]} Array of Maps detailing the resource utilization requests * @async */ - async getStatistics() { + async getStatistics(testLabel) { this.endTime = Date.now()/1000; if (this.include) { const resourceStats = []; - + const chartArray = []; for (const metricKey of Object.keys(this.include)) { let newKey = true; // Each metric is of the form @@ -144,23 +147,42 @@ class PrometheusMonitor extends MonitorInterface { // Retrieve base mapped statistics and coerce into correct format const resultMap = PrometheusQueryHelper.extractStatisticFromRange(response, params.statistic, params.label); + const metricArray = []; for (const [key, value] of resultMap.entries()) { // Filter here if (this.ignore.includes(key)) { continue; } else { - // Transfer into display array + // Build report table information const watchItemStat = newKey ? this.getResultColumnMapForQueryTag(params.query, metricKey) : this.getResultColumnMapForQueryTag('', ''); watchItemStat.set('Name', key); const multiplier = params.multiplier ? params.multiplier : 1; - watchItemStat.set('Value', (value*multiplier).toFixed(2)); + watchItemStat.set('Value', (value*multiplier).toPrecision(this.precision)); // Store resourceStats.push(watchItemStat); newKey = false; + + // Build separate charting information + const metricMap = new Map(); + metricMap.set('Name', watchItemStat.get('Name')); + metricMap.set(metricKey, watchItemStat.get('Value')); + metricArray.push(metricMap); } } + chartArray.push(metricArray); } - return resourceStats; + + // Retrieve Chart data + const chartTypes = this.monitorConfig.charting; + const chartStats = []; + if (chartTypes) { + for (const metrics of chartArray) { + const stats = ChartBuilder.retrieveChartStats(this.constructor.name, chartTypes, `${testLabel}_${metrics[0].get('Name')}`, metrics); + chartStats.push(...stats); + } + } + + return { resourceStats, chartStats }; } else { Logger.debug('No include options specified for monitor - skipping action'); } diff --git a/packages/caliper-core/lib/master/monitor/monitor-utilities.js b/packages/caliper-core/lib/master/monitor/monitor-utilities.js index c10e2907c..6ffb19229 100644 --- a/packages/caliper-core/lib/master/monitor/monitor-utilities.js +++ b/packages/caliper-core/lib/master/monitor/monitor-utilities.js @@ -15,6 +15,8 @@ 'use strict'; +const ConfigUtil = require('../../common/config/config-util'); + /** * Static utility methods for monitors */ @@ -64,6 +66,64 @@ class MonitorUtilities { } } + /** + * Normalize all data in the passed Array of Map items + * @param {String} stat the stat to work on + * @param {Map[]} watchItemStats the full array of items to work on + */ + static normalizeStats(stat, watchItemStats) { + // Collect and determine largest value to normalize to + let maxValue = 0; + const values = []; + for (const watchItem of watchItemStats) { + const value = watchItem.get(stat); + values.push(value); + if (!isNaN(value) && value > maxValue) { + maxValue = value; + } + } + + // Determine divisor and new title + let divisor = 1; + let newStat; + let kb = 1024; + let mb = kb * 1024; + let gb = mb * 1024; + if (maxValue < kb) { + // Bytes + newStat = `${stat} [B]`; + } else if (maxValue < mb) { + // KB + newStat = `${stat} [KB]`; + divisor = kb; + } else if(maxValue < gb) { + // MB + newStat = `${stat} [MB]`; + divisor = mb; + } else { + // GB + newStat = `${stat} [GB]`; + divisor = gb; + } + + // Normalize values + const precision = ConfigUtil.get(ConfigUtil.keys.Report.Precision, 3); + const normValues = []; + for (const value of values) { + if(isNaN(value)) { + normValues.push('-'); + } else { + normValues.push((value / divisor).toPrecision(precision)); + } + } + + for (const watchItem of watchItemStats) { + const modVal = normValues.shift(); + watchItem.set(newStat, modVal); + watchItem.delete(stat); + } + } + /** * Get statistics(maximum, minimum, summation, average) of a number array * @param {Array} arr array of numbers diff --git a/packages/caliper-core/lib/master/report/report-builder.js b/packages/caliper-core/lib/master/report/report-builder.js index 1396bfdc5..1934e470d 100644 --- a/packages/caliper-core/lib/master/report/report-builder.js +++ b/packages/caliper-core/lib/master/report/report-builder.js @@ -12,12 +12,11 @@ * limitations under the License. */ - 'use strict'; const Config = require('../../common/config/config-util'); const Utils = require('../../common/utils/caliper-utils'); -const Logger = Utils.getLogger('report-builder'); +const Logger = Utils.getLogger('caliper-flow'); const fs = require('fs'); const Mustache = require('mustache'); const path = require('path'); @@ -56,27 +55,58 @@ class ReportBuilder { 'head' : [], // table head for the summary table, e.g. ["Test","Name","Succ","Fail","Send Rate","Max Delay","Min Delay", "Avg Delay", "Throughput"], 'results':[] // table rows for the summary table, e.g. [{ "result": [0,"publish", 1,0,"1 tps", "10.78 s","10.78 s", "10.78 s", "1 tps"]},...] }, - // results of each rounds, e.g - // [{ - // "id": "round 0", - // "description" : "Test the performance for publishing digital item", - // "performance": { - // "head": ["Name","Succ","Fail","Send Rate","Max Delay","Min Delay", "Avg Delay", "Throughput"], - // "result": ["publish", 1,0,"1 tps", "10.78 s","10.78 s", "10.78 s", "1 tps"] - // }, - // "resource": { - // "head": ["TYPE","NAME", "Memory(max)","Memory(avg)","CPU(max)", "CPU(avg)", "Traffic In","Traffic Out"], - // "results": [ - // { - // "result": ["Docker","peer1.org1.example.com", "94.4MB", "94.4MB", "0.89%", "0.84%", "0B", "0B"] - // }, - // { - // "result": ["Docker","peer0.org2.example.com","90.4MB", "90.4MB", "1.2%", "1.2%", "6KB", "0B"] - // } - // ] - // } - // } 'tests' : [], + // Each entry in `tests` are the results of each rounds, e.g + // tests: [ + // rounds: [ + // { + // "id": "round 0", + // "description" : "Test the performance for publishing digital item", + // "performance": { + // "head": ["Name","Succ","Fail","Send Rate","Max Delay","Min Delay", "Avg Delay", "Throughput"], + // "result": ["publish", 1,0,"1 tps", "10.78 s","10.78 s", "10.78 s", "1 tps"] + // }, + // "resources": [ + // { + // "monitor": monitor_type (docker | process | prometheus) + // "head": ["TYPE","NAME", "Memory(max)","Memory(avg)","CPU(max)", "CPU(avg)", "Traffic In","Traffic Out"], + // "results": [ + // { + // "result": ["Docker","peer1.org1.example.com", "94.4MB", "94.4MB", "0.89%", "0.84%", "0B", "0B"] + // }, + // { + // "result": ["Docker","peer0.org2.example.com","90.4MB", "90.4MB", "1.2%", "1.2%", "6KB", "0B"] + // } + // ] + // }, + // { + // "monitor": monitor_type (docker | process | prometheus) + // "head": ["TYPE","NAME","Memory(avg)","CPU(max)", "CPU(avg)", "Disc Write","Disc Read"], + // "results": [ + // { + // "result": ["Docker","peer1.org1.example.com", "94.4MB", "9", "0.89", "0.84%", "0B"] + // }, + // { + // "result": ["Docker","peer0.org2.example.com","90.4MB", "9", "1.2", "1.2%", "6KB"] + // } + // ], + // "charts": [ + // { + // "labels": ["peer1.org1.example.com","peer0.org2.example.com"], + // "chart-id": "memory-chart", + // "title": "Memory Avg (MB)", + // "data": [94.4, 90.1] + // }, + // { + // "labels": ["peer1.org1.example.com","peer0.org2.example.com"], + // "chart-id": "cpu-chart", + // "title": "CPU Avg (%)", + // "data": [23, 18] + // } + // ] + // }], + // {...} + // ] 'benchmarkInfo': 'not provided', // readable information for the benchmark 'sut': { 'meta' : [], // metadata of the SUT @@ -125,7 +155,7 @@ class ReportBuilder { * add a row to the summary table * @param {Array} row elements of the row */ - addSummarytableRow(row) { + addSummaryTableRow(row) { if(!Array.isArray(row) || row.length < 1) { throw new Error('unrecognized report row'); } @@ -134,38 +164,50 @@ class ReportBuilder { /** * add a new benchmark round - * @param {String} label the test label + * @param {Object} roundConfig the test round configuration object * @return {Number} id of the round, used to add details to this round */ - addBenchmarkRound(label) { + addBenchmarkRound(roundConfig) { let index; let exists = false; for (let i=0; i 0) { - // print to console for view and add to report Logger.info(`### ${type} resource stats ###'`); this.printTable(resourceTable); - this.reportBuilder.setRoundResource(label, idx, resourceTable); + this.reportBuilder.setRoundResourceTable(label, idx, resourceTable, chartStats, type); } } } @@ -319,14 +309,14 @@ class Report { * Augment the report with details generated through extraction from Prometheus * @param {JSON} timing JSON object containing start/end times required to query * the prometheus server - * @param {String} label label of the test round + * @param {Object} roundConfig test round configuration object * @param {number} round the current test round * @return {Promise} promise object containing the report index * @async */ - async processPrometheusTPSResults(timing, label, round){ + async processPrometheusTPSResults(timing, roundConfig, round){ try { - const resultMap = await this.getPrometheusResultValues(label, round, timing.start, timing.end); + const resultMap = await this.getPrometheusResultValues(roundConfig.label, round, timing.start, timing.end); // Add this set of results to the main round collection this.resultsByRound.push(resultMap); @@ -337,8 +327,8 @@ class Report { this.printTable(tableArray); // Add TPS to the report - let idx = this.reportBuilder.addBenchmarkRound(label); - this.reportBuilder.setRoundPerformance(label, idx, tableArray); + let idx = this.reportBuilder.addBenchmarkRound(roundConfig); + this.reportBuilder.setRoundPerformance(roundConfig.label, idx, tableArray); return idx; } catch (error) { diff --git a/packages/caliper-core/lib/master/report/template/report.html b/packages/caliper-core/lib/master/report/template/report.html index 5d499d5c3..a7937bcb7 100755 --- a/packages/caliper-core/lib/master/report/template/report.html +++ b/packages/caliper-core/lib/master/report/template/report.html @@ -1,4 +1,25 @@ + + Hyperledger Caliper Report @@ -7,38 +28,34 @@ .left-column { position: fixed; width:20%; - border-radius: 5px; - background-color: #f2f2f2; - -webkit-border-radius: 5px; - -moz-border-radius: 5px; - -khtml-border-radius: 5px; } - .right-column { - margin-left: 22%; - width:60%; - } - .left-column ul,h2 { + .left-column ul { display: block; - margin: 10px; padding: 0; list-style: none; - } - .left-column ul{ - border-top: 1px solid #d9d9d9; + border-bottom: 1px solid #d9d9d9; font-size: 14px; } + .left-column h2{ + font-size: 24px; + font-weight: 400; + margin-block-end: 0.5em; + } + .left-column h3{ + font-size: 18px; + font-weight: 400; + margin-block-end: 0.5em; + } .left-column li{ + margin-left: 10px; margin-bottom: 5px; color: #5e6b73; } - .left-column h3 { - border-left: 5px solid #009a61; - } - .right-column>div { - border-top: 1px solid #d9d9d9; + .right-column { + margin-left: 22%; + width:60%; } .right-column table { - font-family: verdana,arial,sans-serif; font-size:11px; color:#333333; border-width: 1px; @@ -46,8 +63,19 @@ border-collapse: collapse; margin-bottom: 10px; } + .right-column h2{ + font-weight: 400; + } + .right-column h3{ + font-weight: 400; + } + .right-column h4 { + font-weight: 400; + margin-block-end: 0; + } .right-column th { border-width: 1px; + font-size: small; padding: 8px; border-style: solid; border-color: #666666; @@ -55,24 +83,16 @@ } .right-column td { border-width: 1px; + font-size: small; padding: 8px; border-style: solid; border-color: #666666; background-color: #ffffff; + font-weight: 400; } .tag { - display: inline-block; margin-bottom: 10px; padding: 5px 10px; - background-color: rgba(1,126,102,0.08); - font-family: verdana,arial,sans-serif; - font-size:11px; - color: #017E66; - text-align: center; - border-radius: 5px; - -webkit-border-radius: 5px; - -moz-border-radius: 5px; - -khtml-border-radius: 5px; } pre { padding: 10px; @@ -86,101 +106,120 @@ max-height:300px; font-size:12px; } + .charting { + display:flex; + flex-direction:row; + flex-wrap: wrap; + page-break-inside: auto; + } + .chart { + display:flex; + flex:1; + max-width: 50%; + } - - -
-
-

Caliper Report

-
    -

     Basic information

    - {{#summary.meta}} -
  • {{name}}:  {{value}}
    • - {{/summary.meta}} -
  • Details
    • -
- -
    -

     System Under Test

    - {{#sut.meta}} -
  • {{name}}:  {{value}}
    • - {{/sut.meta}} -
  • Details
    • -
-
- -
-
- {{#summary}} - -

Summary

- - {{#head}} {{/head}} - - - {{#results}} - - {{#result}} {{/result}} - - {{/results}} -
{{.}}
{{.}}
- {{/summary}} + +
+
+ +
    +

     Basic information

    + {{#summary.meta}} +
  • {{name}}:  {{value}}
    • + {{/summary.meta}} +
  • Details
    • +
+ +
    +

     System under test

    + {{#sut.meta}} +
  • {{name}}:  {{value}}
    • + {{/sut.meta}} +
  • Details
    • +
- {{#tests}} -
-

{{label}}

-

{{description}}

- {{#rounds}} -

{{id}}

- Performance metrics - {{#performance}} - + +
+

Caliper report

+
+ {{#summary}} +
+

Summary of performance metrics

{{#head}} {{/head}} + + {{#results}} {{#result}} {{/result}} + {{/results}}
{{.}}
{{.}}
- {{/performance}} - Resource consumption - {{#resources}} - - -
- - {{#head}} {{/head}} - - {{#results}} - - {{#result}} {{/result}} - - {{/results}} -
{{.}}
{{.}}
- - - {{/resources}} - {{/rounds}} -
- {{/tests}} -
-

Test Environment

- benchmark config - 
{{benchmarkInfo}}
- SUT - 
{{sut.details}}
-
-
- - -
+ {{/summary}} + + {{#tests}} +
+

Benchmark round: {{label}}

+

{{description}}

+ 
{{config}}
+ {{#rounds}} +

Performance metrics for {{label}}

+ {{#performance}} + + + {{#head}} {{/head}} + + + {{#result}} {{/result}} + +
{{.}}
{{.}}
+ {{/performance}} +

Resource utilization for {{label}}

+ {{#resources}} +

Resource monitor: {{monitor}}

+ + +
+ + {{#head}} {{/head}} + + {{#results}} + + {{#result}} {{/result}} + + {{/results}} +
{{.}}
{{.}}
+ + +
+ {{#charts}} +
+ + + +
+ {{/charts}} +
+ {{/resources}} + {{/rounds}} +
+ {{/tests}} +
+

Test Environment

+

benchmark config

+ 
{{benchmarkInfo}}
+

SUT

+ 
{{sut.details}}
+
+ + diff --git a/packages/caliper-core/lib/master/test-runners/round-orchestrator.js b/packages/caliper-core/lib/master/test-runners/round-orchestrator.js index fd3b84ce6..2573c091c 100644 --- a/packages/caliper-core/lib/master/test-runners/round-orchestrator.js +++ b/packages/caliper-core/lib/master/test-runners/round-orchestrator.js @@ -190,9 +190,9 @@ class RoundOrchestrator { // - TPS let idx; if (this.monitorOrchestrator.hasMonitor('prometheus')) { - idx = await this.report.processPrometheusTPSResults({start, end}, roundConfig.label, index); + idx = await this.report.processPrometheusTPSResults({start, end}, roundConfig, index); } else { - idx = await this.report.processLocalTPSResults(results, roundConfig.label); + idx = await this.report.processLocalTPSResults(results, roundConfig); } // - Resource utilization @@ -214,8 +214,6 @@ class RoundOrchestrator { } } - let benchEndTime = Date.now(); - // clean up, with "silent" failure handling try { this.report.printResultsByRound(); @@ -236,6 +234,7 @@ class RoundOrchestrator { logger.error(`Error while stopping clients: ${err.stack || err}`); } + let benchEndTime = Date.now(); logger.info(`Benchmark finished in ${(benchEndTime - benchStartTime)/1000.0} seconds. Total rounds: ${success + failed}. Successful rounds: ${success}. Failed rounds: ${failed}.`); } } diff --git a/packages/caliper-core/package.json b/packages/caliper-core/package.json index 0ebfcc6d1..ae092ef8c 100644 --- a/packages/caliper-core/package.json +++ b/packages/caliper-core/package.json @@ -19,6 +19,7 @@ "npm": ">=5.6.0" }, "dependencies": { + "color-scheme": "^1.0.1", "compare-versions": "^3.4.0", "dockerode": "^2.5.0", "fs-extra": "^8.0.1", diff --git a/packages/caliper-core/test/master/charts/chart-builder.js b/packages/caliper-core/test/master/charts/chart-builder.js new file mode 100644 index 000000000..5e30a5fd8 --- /dev/null +++ b/packages/caliper-core/test/master/charts/chart-builder.js @@ -0,0 +1,292 @@ +/* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +'use strict'; +const rewire = require('rewire'); +const ChartBuilder = require('../../../lib/master/charts/chart-builder'); +const ChartBuilderRewire = rewire('../../../lib/master/charts/chart-builder'); + +const chai = require('chai'); +chai.should(); + +const sinon = require('sinon'); + +describe('chart builder implementation', () => { + + let sandbox; + let revert; + let FakeLogger; + + const defaultType = 'bar'; + const testMonitor = 'CallingMonitor'; + + const resource0 = new Map(); + resource0.set('Name', 'resource0'); + resource0.set('item0 [MB]', '510'); + resource0.set('item1 [MB]', '1e4'); + + const resource1 = new Map(); + resource1.set('Name', 'resource1'); + resource1.set('item0 [MB]', '4.77e-8'); + resource1.set('item1 [MB]', '23'); + + const resource2 = new Map(); + resource2.set('Name', 'resource2'); + resource2.set('item0 [MB]', '100'); + resource2.set('item1 [MB]', '24'); + + describe('#retrieveIncludedMetrics', () => { + + const resource0 = new Map(); + resource0.set('Name', 'resource0'); + resource0.set('item0 [MB]', '53'); + resource0.set('item1 [MB]', '876'); + + beforeEach(() => { + revert = []; + sandbox = sinon.createSandbox(); + + FakeLogger = { + debug: () => { + }, + error: () => { + }, + warn: () => { + } + }; + sandbox.stub(FakeLogger); + }); + + afterEach(() => { + if (revert.length) { + revert.forEach(Function.prototype.call, Function.prototype.call); + } + sandbox.restore(); + }); + + it('should log an error and return an empty array if not provided any metrics', () => { + revert.push(ChartBuilderRewire.__set__('Logger', FakeLogger)); + const all = ChartBuilderRewire.retrieveIncludedMetrics(testMonitor, defaultType, undefined, resource0); + + // all should be empty + all.length.should.equal(0); + sinon.assert.called(FakeLogger.error); + sinon.assert.calledWith(FakeLogger.error, 'Required "metrics" not provided for bar chart generation for monitor CallingMonitor'); + + }); + + it('should log an error and return an empty array if the "all" option is listed with other metrics', () => { + revert.push(ChartBuilderRewire.__set__('Logger', FakeLogger)); + const all = ChartBuilderRewire.retrieveIncludedMetrics(testMonitor, defaultType, ['all', 'anotherMetric'], resource0); + + // all should be empty + all.length.should.equal(0); + sinon.assert.called(FakeLogger.error); + sinon.assert.calledWith(FakeLogger.error, 'Cannot list "all" option with other metrics for bar chart generation for monitor CallingMonitor'); + }); + + it('should provide all metrics if the `all` option is provided', () => { + const all = ChartBuilder.retrieveIncludedMetrics(testMonitor, defaultType, ['all'], resource0); + all.should.deep.equal(['item0 [MB]', 'item1 [MB]']); + + }); + + it('should provide filtered metrics if a named list is provided option is provided', () => { + const all = ChartBuilder.retrieveIncludedMetrics(testMonitor, defaultType, ['item1'], resource0); + all.should.deep.equal(['item1 [MB]']); + + }); + }); + + describe('ChartBuilder.retrieveChartStats', () => { + + beforeEach(() => { + revert = []; + sandbox = sinon.createSandbox(); + + FakeLogger = { + debug: () => { + }, + error: () => { + }, + warn: () => { + } + }; + sandbox.stub(FakeLogger); + }); + + afterEach(() => { + if (revert.length) { + revert.forEach(Function.prototype.call, Function.prototype.call); + } + sandbox.restore(); + }); + + it('should call "barChart" if requested', () => { + const myBarStub = sinon.stub().returns([]); + sandbox.replace(ChartBuilder, 'barChart', myBarStub); + const myPolarStub = sinon.stub().returns([]); + sandbox.replace(ChartBuilder, 'polarChart', myPolarStub); + + const resources = [resource0, resource1, resource2]; + const chartTypes = {bar: {metrics: ['all']}}; + ChartBuilder.retrieveChartStats(testMonitor, chartTypes, 'TEST', resources); + + sinon.assert.calledOnce(myBarStub); + sinon.assert.notCalled(myPolarStub); + }); + + it('should call "polarChart" if requested', () => { + const myBarStub = sinon.stub().returns([]); + sandbox.replace(ChartBuilder, 'barChart', myBarStub); + const myPolarStub = sinon.stub().returns([]); + sandbox.replace(ChartBuilder, 'polarChart', myPolarStub); + + const resources = [resource0, resource1, resource2]; + const chartTypes = {polar: {metrics: ['all']}}; + ChartBuilder.retrieveChartStats(testMonitor, chartTypes, 'TEST', resources); + + sinon.assert.calledOnce(myPolarStub); + sinon.assert.notCalled(myBarStub); + + }); + + it('should call all chart types passed', () => { + const myBarStub = sinon.stub().returns([]); + sandbox.replace(ChartBuilder, 'barChart', myBarStub); + const myPolarStub = sinon.stub().returns([]); + sandbox.replace(ChartBuilder, 'polarChart', myPolarStub); + + const resources = [resource0, resource1, resource2]; + const chartTypes = {bar: {metrics: ['all']}, polar: {metrics: ['all']}}; + ChartBuilder.retrieveChartStats(testMonitor, chartTypes, 'TEST', resources); + + sinon.assert.calledOnce(myBarStub); + sinon.assert.calledOnce(myPolarStub); + + }); + it('should call log error and return empty array if unknown chart type', () => { + revert.push(ChartBuilderRewire.__set__('Logger', FakeLogger)); + + const resources = [resource0, resource1, resource2]; + const chartTypes = {unknown: {metrics: ['all']}}; + + const output = ChartBuilderRewire.retrieveChartStats(testMonitor, chartTypes, 'TEST', resources); + + output.length.should.equal(0); + sinon.assert.called(FakeLogger.error); + sinon.assert.calledWith(FakeLogger.error, 'Unknown chart type named "unknown" requested'); + + }); + + it('should return an array of all items when the `all` option is specified', () => { + + const resources = [resource0, resource1, resource2]; + const chartTypes = {bar: {metrics: ['all']}}; + const chart = ChartBuilder.retrieveChartStats(testMonitor, chartTypes, 'TEST', resources); + + // expect 2 metric charts + chart.length.should.equal(2); + + // chart should have correct structure + chart[0]['chart-id'].should.equal('CallingMonitor_TEST_horizontalBar0'); + const chartData0 = JSON.parse(chart[0]['chart-data']); + chartData0.type.should.equal('horizontalBar'); + chartData0.title.should.equal('item0 [MB]'); + chartData0.legend.should.equal(false); + chartData0.labels.should.deep.equal(['resource0','resource1','resource2']); + chartData0.datasets[0].data.should.deep.equal(['510','4.77e-8','100']); + + + chart[1]['chart-id'].should.equal('CallingMonitor_TEST_horizontalBar1'); + const chartData1 = JSON.parse(chart[1]['chart-data']); + chartData1.type.should.equal('horizontalBar'); + chartData1.title.should.equal('item1 [MB]'); + chartData1.legend.should.equal(false); + chartData1.labels.should.deep.equal(['resource0','resource1','resource2']); + chartData1.datasets[0].data.should.deep.equal(['1e4','23','24']); + }); + + it('should return an array of filtered items when named options are specified', () => { + + const resources = [resource0, resource1, resource2]; + const chartTypes = {bar: {metrics: ['item1']}}; + const chart = ChartBuilder.retrieveChartStats(testMonitor, chartTypes, 'TEST', resources); + + // one metric chart + chart.length.should.equal(1); + + // chart should have correct structure + chart[0]['chart-id'].should.equal('CallingMonitor_TEST_horizontalBar0'); + const chartData0 = JSON.parse(chart[0]['chart-data']); + chartData0.type.should.equal('horizontalBar'); + chartData0.title.should.equal('item1 [MB]'); + chartData0.legend.should.equal(false); + chartData0.labels.should.deep.equal(['resource0','resource1','resource2']); + chartData0.datasets[0].data.should.deep.equal(['1e4','23','24']); + }); + }); + + describe('ChartBuilder.barChart', () => { + + const testLabel = 'testLabel'; + const include = {}; + + beforeEach(() => { + sandbox = sinon.createSandbox(); + }); + + afterEach(() => { + sandbox.restore(); + }); + + it('should call through to _basicChart with "horizontalBar" and no labels', () => { + const myStub = sinon.stub().returns([]); + sandbox.replace(ChartBuilder, '_basicChart', myStub); + + const resources = [resource0, resource1, resource2]; + ChartBuilder.barChart(testMonitor, testLabel, include, resources); + + + sinon.assert.calledOnce(myStub); + myStub.getCall(0).args.should.deep.equal([testMonitor, testLabel, include, resources, 'horizontalBar', false]); + }); + }); + + describe('ChartBuilder.polarChart', () => { + + const testLabel = 'testLabel'; + const include = {}; + + beforeEach(() => { + sandbox = sinon.createSandbox(); + }); + + afterEach(() => { + sandbox.restore(); + }); + + it('should call through to _basicChart with "polarArea" and labels', () => { + const myStub = sinon.stub().returns([]); + sandbox.replace(ChartBuilder, '_basicChart', myStub); + + const resources = [resource0, resource1, resource2]; + ChartBuilder.polarChart(testMonitor, testLabel, include, resources); + + + sinon.assert.calledOnce(myStub); + myStub.getCall(0).args.should.deep.equal([testMonitor, testLabel, include, resources, 'polarArea', true]); + }); + }); +}); diff --git a/packages/caliper-core/test/master/monitor/monitor-utilities.js b/packages/caliper-core/test/master/monitor/monitor-utilities.js new file mode 100644 index 000000000..7fdb28fcd --- /dev/null +++ b/packages/caliper-core/test/master/monitor/monitor-utilities.js @@ -0,0 +1,174 @@ +/* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +'use strict'; + +const Utilities = require('../../../lib/master/monitor/monitor-utilities'); + +const mocha = require('mocha'); +const fail = mocha.fail; +const chai = require('chai'); +chai.should; + +describe('Monitor utilities', () => { + + describe('#Utilities.normalizeStats', () => { + + let valuesMap0; + let valuesMap1; + let valuesMap2; + let mapArray; + + beforeEach(() => { + // Dummy data + valuesMap0 = new Map(); + valuesMap0.set('CPU', 500); + valuesMap0.set('Memory', 80000000); + valuesMap0.set('Disc', 200000000000); + + valuesMap1 = new Map(); + valuesMap1.set('CPU', 70000); + valuesMap1.set('Memory', 100000); + valuesMap1.set('Disc', 100000000000); + + valuesMap2 = new Map(); + valuesMap2.set('CPU', 4500); + valuesMap2.set('Memory', 500000); + valuesMap2.set('Disc', 300000000000); + + mapArray = [valuesMap0, valuesMap1, valuesMap2]; + }); + + it('should normalize all values to the hightest KB value present in the passed map', () => { + + // Run static method + Utilities.normalizeStats('CPU', mapArray); + + // Values for all 'CPU' items should be normalized to 70000 bytes + // We are also changing the name of the metric to include a unit, so we need to check what that might be too + + let newName; + for (const name of valuesMap0.keys()) { + if (name.includes('CPU')) { + newName = name; + break; + } + } + + // Should have modified to be 'CPU [KB]' + if (!newName) { + fail('Unable to determine modified name within Map array'); + } else { + newName.should.equal('CPU [KB]'); + } + + // Values should have been modified to ref + valuesMap0.get(newName).should.equal('0.488'); + valuesMap1.get(newName).should.equal('68.4'); + valuesMap2.get(newName).should.equal('4.39'); + + }); + + it('should normalize all values to the hightest MB value present in the passed map', () => { + // Run static method + Utilities.normalizeStats('Memory', mapArray); + + // Values for all 'CPU' items should be normalized to 70000 bytes + // We are also changing the name of the metric to include a unit, so we need to check what that might be too + + let newName; + for (const name of valuesMap0.keys()) { + if (name.includes('Memory')) { + newName = name; + break; + } + } + + // Should have modified to be 'Memory [MB]' + if (!newName) { + fail('Unable to determine modified name within Map array'); + } else { + newName.should.equal('Memory [MB]'); + } + + // Values should have been modified to ref + valuesMap0.get(newName).should.equal('76.3'); + valuesMap1.get(newName).should.equal('0.0954'); + valuesMap2.get(newName).should.equal('0.477'); + }); + + it('should normalize all values to the hightest GB value present in the passed map', () => { + // Run static method + Utilities.normalizeStats('Disc', mapArray); + + // Values for all 'CPU' items should be normalized to 70000 bytes + // We are also changing the name of the metric to include a unit, so we need to check what that might be too + + let newName; + for (const name of valuesMap0.keys()) { + if (name.includes('Disc')) { + newName = name; + break; + } + } + + // Should have modified to be 'Disc [GB]' + if (!newName) { + fail('Unable to determine modified name within Map array'); + } else { + newName.should.equal('Disc [GB]'); + } + + // Values should have been modified to ref + valuesMap0.get(newName).should.equal('186'); + valuesMap1.get(newName).should.equal('93.1'); + valuesMap2.get(newName).should.equal('279'); + }); + + it('should return a normalized value that includes at least one significant figure', () => { + + // New item with small relative value + const valuesMap3 = new Map(); + valuesMap3.set('Memory', 0.05); + + const newArray = [...mapArray, valuesMap3]; + // Run static method + Utilities.normalizeStats('Memory', newArray); + + // Check for modified name + let newName; + for (const name of valuesMap0.keys()) { + if (name.includes('Memory')) { + newName = name; + break; + } + } + + // Should have modified to be 'Memory [MB]' + if (!newName) { + fail('Unable to determine modified name within Map array'); + } else { + newName.should.equal('Memory [MB]'); + } + + // Values should have been modified to ref + valuesMap0.get(newName).should.equal('76.3'); + valuesMap1.get(newName).should.equal('0.0954'); + valuesMap2.get(newName).should.equal('0.477'); + valuesMap3.get(newName).should.equal('4.77e-8'); + }); + + }); + +}); diff --git a/packages/caliper-tests-integration/fabric_tests/phase2/benchconfig.yaml b/packages/caliper-tests-integration/fabric_tests/phase2/benchconfig.yaml index 2b994378d..c4db4ca1a 100644 --- a/packages/caliper-tests-integration/fabric_tests/phase2/benchconfig.yaml +++ b/packages/caliper-tests-integration/fabric_tests/phase2/benchconfig.yaml @@ -36,4 +36,5 @@ observer: monitor: interval: 1 type: ['process'] - process: [{ command: 'node', arguments: 'fabricClientWorker.js', multiOutput: 'avg' }] + process: + processes: [{ command: 'node', arguments: 'fabricClientWorker.js', multiOutput: 'avg' }] diff --git a/packages/caliper-tests-integration/fabric_tests/phase3/benchconfig.yaml b/packages/caliper-tests-integration/fabric_tests/phase3/benchconfig.yaml index 1f8b5870d..fd895c902 100644 --- a/packages/caliper-tests-integration/fabric_tests/phase3/benchconfig.yaml +++ b/packages/caliper-tests-integration/fabric_tests/phase3/benchconfig.yaml @@ -36,6 +36,7 @@ observer: monitor: interval: 1 type: ['process', 'docker'] - process: [{ command: 'node', arguments: 'fabricClientWorker.js', multiOutput: 'avg' }] + process: + processes: [{ command: 'node', arguments: 'fabricClientWorker.js', multiOutput: 'avg' }] docker: - name: ['peer0.org1.example.com', 'peer0.org2.example.com', 'orderer0.example.com', 'orderer1.example.com'] + containers: ['peer0.org1.example.com', 'peer0.org2.example.com', 'orderer0.example.com', 'orderer1.example.com'] diff --git a/packages/caliper-tests-integration/fabric_tests/phase4/benchconfig.yaml b/packages/caliper-tests-integration/fabric_tests/phase4/benchconfig.yaml index b19c38398..b75227f79 100644 --- a/packages/caliper-tests-integration/fabric_tests/phase4/benchconfig.yaml +++ b/packages/caliper-tests-integration/fabric_tests/phase4/benchconfig.yaml @@ -53,3 +53,8 @@ monitor: label: name statistic: max multiplier: 0.000001 + charting: + polar: + metrics: [Max Memory (MB)] + bar: + metrics: [all] diff --git a/packages/caliper-tests-integration/fabric_tests/phase5/benchconfig.yaml b/packages/caliper-tests-integration/fabric_tests/phase5/benchconfig.yaml index 2b994378d..c4db4ca1a 100644 --- a/packages/caliper-tests-integration/fabric_tests/phase5/benchconfig.yaml +++ b/packages/caliper-tests-integration/fabric_tests/phase5/benchconfig.yaml @@ -36,4 +36,5 @@ observer: monitor: interval: 1 type: ['process'] - process: [{ command: 'node', arguments: 'fabricClientWorker.js', multiOutput: 'avg' }] + process: + processes: [{ command: 'node', arguments: 'fabricClientWorker.js', multiOutput: 'avg' }]