Skip to content

Commit

Permalink
[8.6] [Stack Monitoring] api tests for cluster and elasticsearch (#14…
Browse files Browse the repository at this point in the history 
…5138) (#145985)

# Backport

This will backport the following commits from `main` to `8.6`:
- [[Stack Monitoring] api tests for cluster and elasticsearch
(#145138)](#145138)

<!--- Backport version: 8.9.7 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Kevin
Lacabane","email":"[email protected]"},"sourceCommit":{"committedDate":"2022-11-22T13:02:50Z","message":"[Stack
Monitoring] api tests for cluster and elasticsearch (#145138)\n\n###
Summary\r\nPart of
https://github.com/elastic/kibana/issues/119658\r\n\r\nAdd api
integration tests for cluster and elasticsearch routes to\r\nvalidate
behavior when reading data ingested by elastic-agent.\r\n\r\nWe
currently have a testing suite for legacy and another one
for\r\nmetricbeat. Since metricbeat and agent documents only differ in
their\r\nmetadata, for example agent will populate a `data_stream.*`
property to\r\nidentify the document types while metricbeat uses
`metricset.*`, the\r\ntests assertion validating _business_ data should
pass regardless of the\r\ndocuments source. With this in mind the
metricbeat tests were updated to\r\nrun the tests twice, one time with
metricbeat data and a second time\r\nwith package data.\r\n\r\nTo
generate the archives the `metrics-*` mappings were extracted
with\r\nesArchiver from an elasticsearch with the package installed, and
the\r\ndocuments were transformed from the metricbeat documents with
[this\r\nscript](https://gist.github.com/klacabane/654497ff86053c60af6df15fa6f6f657).\r\n\r\nCo-authored-by:
kibanamachine
<[email protected]>","sha":"5cf0d0f24817db4b36501c6c06f6cfb4cd61c296","branchLabelMapping":{"^v8.7.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["Team:Infra
Monitoring UI","release_note:skip","Feature:Stack
Monitoring","v8.6.0","v8.7.0"],"number":145138,"url":"https://github.com/elastic/kibana/pull/145138","mergeCommit":{"message":"[Stack
Monitoring] api tests for cluster and elasticsearch (#145138)\n\n###
Summary\r\nPart of
https://github.com/elastic/kibana/issues/119658\r\n\r\nAdd api
integration tests for cluster and elasticsearch routes to\r\nvalidate
behavior when reading data ingested by elastic-agent.\r\n\r\nWe
currently have a testing suite for legacy and another one
for\r\nmetricbeat. Since metricbeat and agent documents only differ in
their\r\nmetadata, for example agent will populate a `data_stream.*`
property to\r\nidentify the document types while metricbeat uses
`metricset.*`, the\r\ntests assertion validating _business_ data should
pass regardless of the\r\ndocuments source. With this in mind the
metricbeat tests were updated to\r\nrun the tests twice, one time with
metricbeat data and a second time\r\nwith package data.\r\n\r\nTo
generate the archives the `metrics-*` mappings were extracted
with\r\nesArchiver from an elasticsearch with the package installed, and
the\r\ndocuments were transformed from the metricbeat documents with
[this\r\nscript](https://gist.github.com/klacabane/654497ff86053c60af6df15fa6f6f657).\r\n\r\nCo-authored-by:
kibanamachine
<[email protected]>","sha":"5cf0d0f24817db4b36501c6c06f6cfb4cd61c296"}},"sourceBranch":"main","suggestedTargetBranches":["8.6"],"targetPullRequestStates":[{"branch":"8.6","label":"v8.6.0","labelRegex":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"main","label":"v8.7.0","labelRegex":"^v8.7.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/145138","number":145138,"mergeCommit":{"message":"[Stack
Monitoring] api tests for cluster and elasticsearch (#145138)\n\n###
Summary\r\nPart of
https://github.com/elastic/kibana/issues/119658\r\n\r\nAdd api
integration tests for cluster and elasticsearch routes to\r\nvalidate
behavior when reading data ingested by elastic-agent.\r\n\r\nWe
currently have a testing suite for legacy and another one
for\r\nmetricbeat. Since metricbeat and agent documents only differ in
their\r\nmetadata, for example agent will populate a `data_stream.*`
property to\r\nidentify the document types while metricbeat uses
`metricset.*`, the\r\ntests assertion validating _business_ data should
pass regardless of the\r\ndocuments source. With this in mind the
metricbeat tests were updated to\r\nrun the tests twice, one time with
metricbeat data and a second time\r\nwith package data.\r\n\r\nTo
generate the archives the `metrics-*` mappings were extracted
with\r\nesArchiver from an elasticsearch with the package installed, and
the\r\ndocuments were transformed from the metricbeat documents with
[this\r\nscript](https://gist.github.com/klacabane/654497ff86053c60af6df15fa6f6f657).\r\n\r\nCo-authored-by:
kibanamachine
<[email protected]>","sha":"5cf0d0f24817db4b36501c6c06f6cfb4cd61c296"}}]}]
BACKPORT-->

Co-authored-by: Kevin Lacabane <[email protected]>
  • Loading branch information
@kibanamachine @klacabane
kibanamachine and klacabane authored Nov 22, 2022
1 parent 93d7303 commit 52133e6
Show file tree
Hide file tree
Showing 23 changed files with 43,775 additions and 675 deletions.
22 changes: 19 additions & 3 deletions x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/ccr.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,10 @@
*/

import { get, groupBy } from 'lodash';
import { prefixIndexPatternWithCcs } from '../../../../../common/ccs_utils';
import { INDEX_PATTERN_ELASTICSEARCH } from '../../../../../common/constants';
import {
getIndexPatterns,
getElasticsearchDataset,
} from '../../../../lib/cluster/get_index_patterns';
import {
postElasticsearchCcrRequestParamsRT,
postElasticsearchCcrRequestPayloadRT,
Expand Down Expand Up @@ -140,6 +142,13 @@ function buildRequest(
},
},
},
{
term: {
'data_stream.dataset': {
value: getElasticsearchDataset('ccr'),
},
},
},
],
},
},
Expand Down Expand Up @@ -215,7 +224,14 @@ export function ccrRoute(server: MonitoringCore) {
async handler(req) {
const config = server.config;
const ccs = req.payload.ccs;
const esIndexPattern = prefixIndexPatternWithCcs(config, INDEX_PATTERN_ELASTICSEARCH, ccs);
const dataset = 'ccr';
const moduleType = 'elasticsearch';
const esIndexPattern = getIndexPatterns({
config,
moduleType,
dataset,
ccs,
});

try {
const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('monitoring');
Expand Down
55 changes: 30 additions & 25 deletions x-pack/test/api_integration/apis/monitoring/cluster/list_mb.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,33 +13,38 @@ export default function ({ getService }) {
const supertest = getService('supertest');
const { setup, tearDown } = getLifecycleMethods(getService);

describe('list mb', function () {
// Archive contains non-cgroup data which collides with the in-cgroup services present by default on cloud deployments
this.tags(['skipCloud']);

describe('with trial license clusters', () => {
const archive = 'x-pack/test/functional/es_archives/monitoring/multicluster_mb';
const timeRange = {
min: '2017-08-15T21:00:00Z',
max: '2017-08-16T00:00:00Z',
};
const codePaths = ['all'];

before('load clusters archive', () => {
return setup(archive);
});
describe('list - metricbeat and package', function () {
['mb', 'package'].forEach((source) => {
describe(`list ${source}`, function () {
// Archive contains non-cgroup data which collides with the in-cgroup services present by default on cloud deployments
this.tags(['skipCloud']);

after('unload clusters archive', () => {
return tearDown();
});
const archive = `x-pack/test/functional/es_archives/monitoring/multicluster_${source}`;

describe('with trial license clusters', () => {
const timeRange = {
min: '2017-08-15T21:00:00Z',
max: '2017-08-16T00:00:00Z',
};
const codePaths = ['all'];

before('load clusters archive', () => {
return setup(archive);
});

after('unload clusters archive', () => {
return tearDown(archive);
});

it('should load multiple clusters', async () => {
const { body } = await supertest
.post('/api/monitoring/v1/clusters')
.set('kbn-xsrf', 'xxx')
.send({ timeRange, codePaths })
.expect(200);
expect(body).to.eql(multiclusterFixture);
it('should load multiple clusters', async () => {
const { body } = await supertest
.post('/api/monitoring/v1/clusters')
.set('kbn-xsrf', 'xxx')
.send({ timeRange, codePaths })
.expect(200);
expect(body).to.eql(multiclusterFixture);
});
});
});
});
});
Expand Down
55 changes: 30 additions & 25 deletions x-pack/test/api_integration/apis/monitoring/cluster/overview_mb.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,33 +13,38 @@ export default function ({ getService }) {
const supertest = getService('supertest');
const { setup, tearDown } = getLifecycleMethods(getService);

describe('overview mb', function () {
// TODO: https://github.com/elastic/stack-monitoring/issues/31
this.tags(['skipCloud']);

describe('with trial license clusters', () => {
const archive = 'x-pack/test/functional/es_archives/monitoring/singlecluster_green_gold_mb';
const timeRange = {
min: '2017-08-23T21:29:35Z',
max: '2017-08-23T21:47:25Z',
};
const codePaths = ['all'];

before('load clusters archive', () => {
return setup(archive);
});
describe('overview - metricbeat and package', function () {
['mb', 'package'].forEach((source) => {
describe(`overview ${source}`, function () {
// TODO: https://github.com/elastic/stack-monitoring/issues/31
this.tags(['skipCloud']);

after('unload clusters archive', () => {
return tearDown();
});
const archive = `x-pack/test/functional/es_archives/monitoring/singlecluster_green_gold_${source}`;

describe('with trial license clusters', () => {
const timeRange = {
min: '2017-08-23T21:29:35Z',
max: '2017-08-23T21:47:25Z',
};
const codePaths = ['all'];

before('load clusters archive', () => {
return setup(archive);
});

after('unload clusters archive', () => {
return tearDown(archive);
});

it('should load multiple clusters', async () => {
const { body } = await supertest
.post('/api/monitoring/v1/clusters/y1qOsQPiRrGtmdEuM3APJw')
.set('kbn-xsrf', 'xxx')
.send({ timeRange, codePaths })
.expect(200);
expect(body).to.eql(overviewFixture);
it('should load multiple clusters', async () => {
const { body } = await supertest
.post('/api/monitoring/v1/clusters/y1qOsQPiRrGtmdEuM3APJw')
.set('kbn-xsrf', 'xxx')
.send({ timeRange, codePaths })
.expect(200);
expect(body).to.eql(overviewFixture);
});
});
});
});
});
Expand Down
48 changes: 26 additions & 22 deletions x-pack/test/api_integration/apis/monitoring/elasticsearch/ccr_mb.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,32 +12,36 @@ import { getLifecycleMethods } from '../data_stream';
export default function ({ getService }) {
const supertest = getService('supertest');

describe('ccr mb', () => {
const archive = 'x-pack/test/functional/es_archives/monitoring/ccr_mb';
const { setup, tearDown } = getLifecycleMethods(getService);
const timeRange = {
min: '2018-09-19T00:00:00.000Z',
max: '2018-09-19T23:59:59.000Z',
};
describe('ccr - metricbeat and package', () => {
['mb', 'package'].forEach((source) => {
describe(`ccr ${source}`, () => {
const archive = `x-pack/test/functional/es_archives/monitoring/ccr_${source}`;
const { setup, tearDown } = getLifecycleMethods(getService);
const timeRange = {
min: '2018-09-19T00:00:00.000Z',
max: '2018-09-19T23:59:59.000Z',
};

before('load archive', () => {
return setup(archive);
});
before('load archive', () => {
return setup(archive);
});

after('unload archive', () => {
return tearDown();
});
after('unload archive', () => {
return tearDown(archive);
});

it('should return all followers and a grouping of stats by follower index', async () => {
const { body } = await supertest
.post('/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/ccr')
.set('kbn-xsrf', 'xxx')
.send({
timeRange,
})
.expect(200);
it('should return all followers and a grouping of stats by follower index', async () => {
const { body } = await supertest
.post('/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/ccr')
.set('kbn-xsrf', 'xxx')
.send({
timeRange,
})
.expect(200);

expect(body).to.eql(ccrFixture);
expect(body).to.eql(ccrFixture);
});
});
});
});
}
59 changes: 32 additions & 27 deletions x-pack/test/api_integration/apis/monitoring/elasticsearch/ccr_shard_mb.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,35 +14,40 @@ export default function ({ getService }) {
const supertest = getService('supertest');
const { setup, tearDown } = getLifecycleMethods(getService);

describe('ccr shard mb', () => {
const archive = 'x-pack/test/functional/es_archives/monitoring/ccr_mb';
const timeRange = {
min: '2018-09-19T00:00:00.000Z',
max: '2018-09-19T23:59:59.000Z',
};

before('load archive', () => {
return setup(archive);
});
describe('ccr shard - metricbeat and package', () => {
['mb', 'package'].forEach((source) => {
const archive = `x-pack/test/functional/es_archives/monitoring/ccr_${source}`;

after('unload archive', () => {
return tearDown();
});
describe(`ccr shard ${source}`, () => {
const timeRange = {
min: '2018-09-19T00:00:00.000Z',
max: '2018-09-19T23:59:59.000Z',
};

before('load archive', () => {
return setup(archive);
});

after('unload archive', () => {
return tearDown(archive);
});

it('should return specific shard details', async () => {
const { body } = await supertest
.post(
'/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/ccr/follower/shard/0'
)
.set('kbn-xsrf', 'xxx')
.send({
timeRange,
})
.expect(200);

it('should return specific shard details', async () => {
const { body } = await supertest
.post(
'/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/ccr/follower/shard/0'
)
.set('kbn-xsrf', 'xxx')
.send({
timeRange,
})
.expect(200);

// These will be inherently different, but they are only shown in JSON format in the UI so that's okay
const keysToIgnore = ['stat', 'oldestStat'];
expect(omit(body, keysToIgnore)).to.eql(omit(ccrShardFixture, keysToIgnore));
// These will be inherently different, but they are only shown in JSON format in the UI so that's okay
const keysToIgnore = ['stat', 'oldestStat'];
expect(omit(body, keysToIgnore)).to.eql(omit(ccrShardFixture, keysToIgnore));
});
});
});
});
}
85 changes: 44 additions & 41 deletions x-pack/test/api_integration/apis/monitoring/elasticsearch/index_detail_mb.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,50 +14,53 @@ export default function ({ getService }) {
const supertest = getService('supertest');
const { setup, tearDown } = getLifecycleMethods(getService);

describe('index detail mb', () => {
const archive =
'x-pack/test/functional/es_archives/monitoring/singlecluster_three_nodes_shard_relocation_mb';
const timeRange = {
min: '2017-10-05T20:31:48.000Z',
max: '2017-10-05T20:35:12.000Z',
};

before('load archive', () => {
return setup(archive);
});
describe('index detail - metricbeat and package', () => {
['mb', 'package'].forEach((source) => {
describe(`index detail ${source}`, () => {
const archive = `x-pack/test/functional/es_archives/monitoring/singlecluster_three_nodes_shard_relocation_${source}`;
const timeRange = {
min: '2017-10-05T20:31:48.000Z',
max: '2017-10-05T20:35:12.000Z',
};

after('unload archive', () => {
return tearDown();
});
before('load archive', () => {
return setup(archive);
});

it('should summarize index with chart metrics data for the non-advanced view', async () => {
const { body } = await supertest
.post(
'/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/indices/avocado-tweets-2017.10.02'
)
.set('kbn-xsrf', 'xxx')
.send({
timeRange,
is_advanced: false,
})
.expect(200);

expect(body).to.eql(indexDetailFixture);
});
after('unload archive', () => {
return tearDown(archive);
});

it('should summarize index with chart metrics data for the non-advanced view', async () => {
const { body } = await supertest
.post(
'/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/indices/avocado-tweets-2017.10.02'
)
.set('kbn-xsrf', 'xxx')
.send({
timeRange,
is_advanced: false,
})
.expect(200);

expect(body).to.eql(indexDetailFixture);
});

it('should summarize index with chart metrics data for the advanced view', async () => {
const { body } = await supertest
.post(
'/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/indices/avocado-tweets-2017.10.02'
)
.set('kbn-xsrf', 'xxx')
.send({
timeRange,
is_advanced: true,
})
.expect(200);

it('should summarize index with chart metrics data for the advanced view', async () => {
const { body } = await supertest
.post(
'/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/indices/avocado-tweets-2017.10.02'
)
.set('kbn-xsrf', 'xxx')
.send({
timeRange,
is_advanced: true,
})
.expect(200);

expect(body).to.eql(indexDetailAdvancedFixture);
expect(body).to.eql(indexDetailAdvancedFixture);
});
});
});
});
}
Loading

0 comments on commit 52133e6

Please sign in to comment.