Skip to content

Commit

Permalink
Caliper terminates if prometheus is not available
Browse files Browse the repository at this point in the history 
This is due to the error event not being correctly captured when a
request is made to prometheus

Also added some extra code to output a warning and stop trying to do any
more queries for the round.

It won't stop it for all rounds but checks on every round.

closes hyperledger-caliper#1267
  • Loading branch information
D committed Mar 24, 2022
1 parent 6a76834 commit a6d3be3
Show file tree
Hide file tree
Showing 3 changed files with 124 additions and 23 deletions.
9 changes: 5 additions & 4 deletions packages/caliper-core/lib/common/prometheus/prometheus-query-client.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,10 +149,11 @@ class PrometheusQueryClient {
resolve();
}
});
res.on('error', err => {
Logger.error(err);
reject(err);
});
});

req.on('error', err => {
Logger.error(err);
reject(err);
});

req.end();
Expand Down
19 changes: 13 additions & 6 deletions packages/caliper-core/lib/manager/monitors/monitor-prometheus.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ class PrometheusMonitor extends MonitorInterface {
* Retrieve the query client
* @returns {PrometheusQueryClient} the query client
*/
getQueryClient(){
getQueryClient() {
return this.prometheusQueryClient;
}

Expand All @@ -74,7 +74,7 @@ class PrometheusMonitor extends MonitorInterface {
* @async
*/
async start() {
this.startTime = Date.now()/1000;
this.startTime = Date.now() / 1000;
}

/**
Expand Down Expand Up @@ -114,7 +114,7 @@ class PrometheusMonitor extends MonitorInterface {
* @async
*/
async getStatistics(testLabel) {
this.endTime = Date.now()/1000;
this.endTime = Date.now() / 1000;

const resourceStats = [];
const chartArray = [];
Expand All @@ -131,7 +131,14 @@ class PrometheusMonitor extends MonitorInterface {
// label: a matching label for the component of interest
// }
const queryString = PrometheusQueryHelper.buildStringRangeQuery(queryObject.query, this.startTime, this.endTime, queryObject.step);
const response = await this.prometheusQueryClient.getByEncodedUrl(queryString);

let response;
try {
response = await this.prometheusQueryClient.getByEncodedUrl(queryString);
} catch (error) {
Logger.warn('Failed to connect to Prometheus, unable to perform queries');
break;
}

// Retrieve map of component names and corresponding values for the issued query
const componentNameValueMap = PrometheusQueryHelper.extractStatisticFromRange(response, queryObject.statistic, queryObject.label);
Expand All @@ -146,13 +153,13 @@ class PrometheusMonitor extends MonitorInterface {
newQueryObjectIteration = false;
watchItemStat.set('Name', key);
const multiplier = queryObject.multiplier ? queryObject.multiplier : 1;
watchItemStat.set('Value', (value*multiplier).toPrecision(this.precision));
watchItemStat.set('Value', (value * multiplier).toPrecision(this.precision));
// Store
resourceStats.push(watchItemStat);

// Build separate charting information
const metricMap = new Map();
metricMap.set('Name', watchItemStat.get('Name'));
metricMap.set('Name', watchItemStat.get('Name'));
metricMap.set(queryObject.name, watchItemStat.get('Value'));
metricArray.push(metricMap);
}
Expand Down
119 changes: 106 additions & 13 deletions packages/caliper-core/test/manager/monitors/monitor-prometheus.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,33 +21,58 @@ const chai = require('chai');
const should = chai.should();
const sinon = require('sinon');

class FakeQueryClient {
static getByEncodedUrlCount = 0;

static reset() {
FakeQueryClient.getByEncodedUrlCount = 0;
}

static setGetByEncodedUrlResponse(ableToConnect, response) {
FakeQueryClient.ableToConnect = ableToConnect;
FakeQueryClient.response = response;
}

async getByEncodedUrl() {
FakeQueryClient.getByEncodedUrlCount++;
if (!FakeQueryClient.ableToConnect) {
throw new Error('ECONNREFUSED');
}
return FakeQueryClient.response;
}
}

describe('Prometheus monitor implementation', () => {

const fakeQueryClient = sinon.stub();
PrometheusMonitorRewire.__set__('PrometheusQueryClient', fakeQueryClient);
PrometheusMonitorRewire.__set__('PrometheusQueryClient', FakeQueryClient);

// Before/After
let clock;
beforeEach( () => {
beforeEach(() => {
clock = sinon.useFakeTimers();
});

afterEach( () => {
afterEach(() => {
clock.restore();
});

// Test data
const monitorOptions = {
metrics : {
include: ['peer', 'pushgateway', 'dev.*'],
url: 'http://localhost:9090',
include: ['peer', 'orderer', 'dev.*'],
queries: [
{
name: 'avg cpu',
label: 'name',
query: 'sum(rate(container_cpu_usage_seconds_total{name=~".+"}[$interval])) by (name) * 100',
statistic: 'average'
statistic: 'avg'
},
{
name: 'max cpu',
label: 'name',
query: 'sum(rate(container_cpu_usage_seconds_total{name=~".+"}[$interval])) by (name) * 100',
statistic: 'maximum'
statistic: 'max'
}
]
}
Expand Down Expand Up @@ -78,7 +103,7 @@ describe('Prometheus monitor implementation', () => {
});
});

describe('#getQueryClient', ()=>{
describe('#getQueryClient', () => {

it('should return the internal Query Client', () => {
const mon = new PrometheusMonitorRewire({});
Expand All @@ -92,7 +117,7 @@ describe('Prometheus monitor implementation', () => {

it('should set the start time with the current time', () => {
clock.tick(42);
const mon = new PrometheusMonitorRewire({push_url: '123'});
const mon = new PrometheusMonitorRewire({ push_url: '123' });
mon.start();
mon.startTime.should.equal(0.042);
});
Expand All @@ -102,7 +127,7 @@ describe('Prometheus monitor implementation', () => {
describe('#stop', () => {
it('should remove startTime if it exists', () => {
clock.tick(42);
const mon = new PrometheusMonitorRewire({push_url: '123'});
const mon = new PrometheusMonitorRewire({ push_url: '123' });
mon.start();
mon.startTime.should.equal(0.042);
mon.stop();
Expand All @@ -114,7 +139,7 @@ describe('Prometheus monitor implementation', () => {

it('should reset the start time', () => {
clock.tick(42);
const mon = new PrometheusMonitorRewire({push_url: '123'});
const mon = new PrometheusMonitorRewire({ push_url: '123' });
mon.start();
clock.tick(42);
mon.restart();
Expand Down Expand Up @@ -149,15 +174,83 @@ describe('Prometheus monitor implementation', () => {
const mon = new PrometheusMonitorRewire(monitorOptions);
mon.includeStatistic('peer0.org0.example.com').should.equal(true);

mon.includeStatistic('pushgateway').should.equal(true);
mon.includeStatistic('pushgateway').should.equal(false);

mon.includeStatistic('dev-org0.example.com').should.equal(true);

mon.includeStatistic('penuin').should.equal(false);

mon.includeStatistic('orderer0.example.com').should.equal(false);
mon.includeStatistic('orderer0.example.com').should.equal(true);
});
});

describe('When getting statistics', () => {
const response = {
status: 'success',
data: {
resultType: 'matrix',
result: [
{
metric: {
name: 'orderer.example.com'
},
values: [
[
1648125000.736,
'37318656'
],
[
1648125010.736,
'37318656'
],
[
1648125020.736,
'37318656'
]
]
},
{
metric: {
name: 'peer0.org1.example.com'
},
values: [
[
1648125000.736,
'80855040'
],
[
1648125010.736,
'80855040'
],
[
1648125020.736,
'80855040'
]
]
}
]
}
};

it('should stop processing further queries and return an empty set of results if it fails to connect to prometheus ', async () => {
const prometheusMonitor = new PrometheusMonitorRewire(monitorOptions);
FakeQueryClient.reset();
FakeQueryClient.setGetByEncodedUrlResponse(false);
const res = await prometheusMonitor.getStatistics();
FakeQueryClient.getByEncodedUrlCount.should.equal(1);
res.resourceStats.length.should.equal(0);
res.chartStats.length.should.equal(0);
});

it('should process all queries successfully if able to connect to prometheus', async () => {
const prometheusMonitor = new PrometheusMonitorRewire(monitorOptions);
FakeQueryClient.reset();
FakeQueryClient.setGetByEncodedUrlResponse(true, response);
const res = await prometheusMonitor.getStatistics();
FakeQueryClient.getByEncodedUrlCount.should.equal(2);
res.resourceStats.length.should.equal(4);
res.chartStats.length.should.equal(0);
});
});

});

0 comments on commit a6d3be3

Please sign in to comment.