[Stack monitoring] adapt ES queries to updated document structure (#1…

…21641)

* adapt ES queries to updated document structure

* lint

* Revert "lint"

This reverts commit 2ad61e4.

* Revert "adapt ES queries to updated document structure"

This reverts commit 25d4357.

* fix cluster state property accesses

* only query index_recovery with ecs compliant indices

* lint

* setup/teardown to support datastreams

* lint

* update _mb archived data to target .monitoring index

* load archive with useCreate for datastream compatibility

* Revert "update _mb archived data to target .monitoring index"

This reverts commit 7bad361.

* ignore non-elasticsearch archived documents

* Revert "ignore non-elasticsearch archived documents"

This reverts commit 26daeb7.

* use correct index for archived documents

* add datastream lifecycle helper for api-integration tests

* use datastream lifecycle methods

* transform ccr_mb archive

* clean up datastreams

* lint

* load _mb data

Co-authored-by: Kibana Machine <[email protected]>

x-pack/plugins/monitoring/common/constants.ts

@@ -123,12 +123,14 @@ export const CLUSTER_ALERTS_ADDRESS_CONFIG_KEY = 'cluster_alerts.email_notificat
 
 export const STANDALONE_CLUSTER_CLUSTER_UUID = '__standalone_cluster__';
 
-export const INDEX_PATTERN = '.monitoring-*-6-*,.monitoring-*-7-*';
-export const INDEX_PATTERN_KIBANA = '.monitoring-kibana-6-*,.monitoring-kibana-7-*';
-export const INDEX_PATTERN_LOGSTASH = '.monitoring-logstash-6-*,.monitoring-logstash-7-*';
-export const INDEX_PATTERN_BEATS = '.monitoring-beats-6-*,.monitoring-beats-7-*';
-export const INDEX_ALERTS = '.monitoring-alerts-6*,.monitoring-alerts-7*';
-export const INDEX_PATTERN_ELASTICSEARCH = '.monitoring-es-6-*,.monitoring-es-7-*';
+export const INDEX_PATTERN = '.monitoring-*';
+export const INDEX_PATTERN_KIBANA = '.monitoring-kibana-*';
+export const INDEX_PATTERN_LOGSTASH = '.monitoring-logstash-*';
+export const INDEX_PATTERN_BEATS = '.monitoring-beats-*';
+export const INDEX_ALERTS = '.monitoring-alerts-*';
+export const INDEX_PATTERN_ELASTICSEARCH = '.monitoring-es-*';
+// ECS-compliant patterns (metricbeat >8 and agent)
+export const INDEX_PATTERN_ELASTICSEARCH_ECS = '.monitoring-es-8-*';
 export const INDEX_PATTERN_ENTERPRISE_SEARCH = '.monitoring-ent-search-*';
 
 // This is the unique token that exists in monitoring indices collected by metricbeat

x-pack/plugins/monitoring/public/components/elasticsearch/shard_activity/parse_props.js

@@ -29,6 +29,7 @@ export const parseProps = (props) => {
     stage,
     index,
     index_name: indexName,
+    name: mbIndexName,
     primary: isPrimary,
     start_time_in_millis: startTimeInMillis,
     total_time_in_millis: totalTimeInMillis,
@@ -42,7 +43,7 @@ export const parseProps = (props) => {
   const { files, size } = index;
 
   return {
-    name: indexName || index.name,
+    name: indexName || mbIndexName,
     shard: `${id} / ${isPrimary ? 'Primary' : 'Replica'}`,
     relocationType: type === 'PRIMARY_RELOCATION' ? 'Primary Relocation' : normalizeString(type),
     stage: normalizeString(stage),

x-pack/plugins/monitoring/server/lib/elasticsearch/get_last_recovery.ts

@@ -89,7 +89,7 @@ export function handleMbLastRecoveries(resp: ElasticsearchResponse, start: numbe
 export async function getLastRecovery(
   req: LegacyRequest,
   esIndexPattern: string,
-  mbIndexPattern: string,
+  esIndexPatternEcs: string,
   size: number
 ) {
   checkParam(esIndexPattern, 'esIndexPattern in elasticsearch/getLastRecovery');
@@ -109,8 +109,8 @@ export async function getLastRecovery(
       query: createQuery({ type: 'index_recovery', start, end, clusterUuid, metric }),
     },
   };
-  const mbParams = {
-    index: mbIndexPattern,
+  const ecsParams = {
+    index: esIndexPatternEcs,
     size,
     ignore_unavailable: true,
     body: {
@@ -130,7 +130,7 @@ export async function getLastRecovery(
   const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('monitoring');
   const [legacyResp, mbResp] = await Promise.all([
     callWithRequest(req, 'search', legacyParams),
-    callWithRequest(req, 'search', mbParams),
+    callWithRequest(req, 'search', ecsParams),
   ]);
   const legacyResult = handleLegacyLastRecoveries(legacyResp, start);
   const mbResult = handleMbLastRecoveries(mbResp, start);

x-pack/plugins/monitoring/server/lib/elasticsearch/shards/get_shard_allocation.ts

@@ -77,7 +77,7 @@ export function getShardAllocation(
           },
           {
             term: {
-              'elasticsearch.cluster.state.id': stateUuid,
+              'elasticsearch.cluster.stats.state.state_uuid': stateUuid,
             },
           },
         ],

x-pack/plugins/monitoring/server/lib/elasticsearch/shards/get_shard_stats.ts

@@ -69,7 +69,8 @@ export function getShardStats(
   } else if (cluster.elasticsearch?.cluster?.stats?.state?.state_uuid) {
     filters.push({
       term: {
-        'elasticsearch.cluster.state.id': cluster.elasticsearch.cluster.stats.state.state_uuid,
+        'elasticsearch.cluster.stats.state.state_uuid':
+          cluster.elasticsearch.cluster.stats.state.state_uuid,
       },
     });
   }

x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/overview.js

@@ -13,7 +13,10 @@ import { getMetrics } from '../../../../lib/details/get_metrics';
 import { handleError } from '../../../../lib/errors/handle_error';
 import { prefixIndexPattern } from '../../../../../common/ccs_utils';
 import { metricSet } from './metric_set_overview';
-import { INDEX_PATTERN_ELASTICSEARCH } from '../../../../../common/constants';
+import {
+  INDEX_PATTERN_ELASTICSEARCH,
+  INDEX_PATTERN_ELASTICSEARCH_ECS,
+} from '../../../../../common/constants';
 import { getLogs } from '../../../../lib/logs';
 import { getIndicesUnassignedShardStats } from '../../../../lib/elasticsearch/shards/get_indices_unassigned_shard_stats';
 
@@ -46,9 +49,9 @@ export function esOverviewRoute(server) {
         ccs,
         true
       );
-      const mbIndexPattern = prefixIndexPattern(
+      const esEcsIndexPattern = prefixIndexPattern(
         config,
-        config.get('monitoring.ui.metricbeat.index'),
+        INDEX_PATTERN_ELASTICSEARCH_ECS,
         ccs,
         true
       );
@@ -70,7 +73,7 @@ export function esOverviewRoute(server) {
           getLastRecovery(
             req,
             esLegacyIndexPattern,
-            mbIndexPattern,
+            esEcsIndexPattern,
             config.get('monitoring.ui.max_bucket_size')
           ),
           getLogs(config, req, filebeatIndexPattern, { clusterUuid, start, end }),

x-pack/test/api_integration/apis/monitoring/data_stream.js

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const getLifecycleMethods = (getService) => {
+  const esArchiver = getService('esArchiver');
+  const client = getService('es');
+
+  const deleteDataStream = async (index) => {
+    await client.transport.request(
+      {
+        method: 'DELETE',
+        path: `_data_stream/${index}`,
+      },
+      {
+        ignore: [404],
+      }
+    );
+  };
+
+  return {
+    async setup(archive) {
+      await esArchiver.load(archive, { useCreate: true });
+    },
+
+    async tearDown() {
+      await deleteDataStream('.monitoring-*');
+    },
+  };
+};

x-pack/test/api_integration/apis/monitoring/elasticsearch/ccr_mb.js

@@ -7,24 +7,25 @@
 
 import expect from '@kbn/expect';
 import ccrFixture from './fixtures/ccr';
+import { getLifecycleMethods } from '../data_stream';
 
 export default function ({ getService }) {
   const supertest = getService('supertest');
-  const esArchiver = getService('esArchiver');
 
   describe('ccr mb', () => {
     const archive = 'x-pack/test/functional/es_archives/monitoring/ccr_mb';
+    const { setup, tearDown } = getLifecycleMethods(getService);
     const timeRange = {
       min: '2018-09-19T00:00:00.000Z',
       max: '2018-09-19T23:59:59.000Z',
     };
 
     before('load archive', () => {
-      return esArchiver.load(archive);
+      return setup(archive);
     });
 
     after('unload archive', () => {
-      return esArchiver.unload(archive);
+      return tearDown();
     });
 
     it('should return all followers and a grouping of stats by follower index', async () => {

x-pack/test/api_integration/apis/monitoring/elasticsearch/ccr_shard_mb.js

@@ -8,10 +8,11 @@
 import { omit } from 'lodash';
 import expect from '@kbn/expect';
 import ccrShardFixture from './fixtures/ccr_shard';
+import { getLifecycleMethods } from '../data_stream';
 
 export default function ({ getService }) {
   const supertest = getService('supertest');
-  const esArchiver = getService('esArchiver');
+  const { setup, tearDown } = getLifecycleMethods(getService);
 
   describe('ccr shard mb', () => {
     const archive = 'x-pack/test/functional/es_archives/monitoring/ccr_mb';
@@ -21,11 +22,11 @@ export default function ({ getService }) {
     };
 
     before('load archive', () => {
-      return esArchiver.load(archive);
+      return setup(archive);
     });
 
     after('unload archive', () => {
-      return esArchiver.unload(archive);
+      return tearDown();
     });
 
     it('should return specific shard details', async () => {

x-pack/test/api_integration/apis/monitoring/elasticsearch/index_detail_mb.js

@@ -8,10 +8,11 @@
 import expect from '@kbn/expect';
 import indexDetailFixture from './fixtures/index_detail';
 import indexDetailAdvancedFixture from './fixtures/index_detail_advanced';
+import { getLifecycleMethods } from '../data_stream';
 
 export default function ({ getService }) {
   const supertest = getService('supertest');
-  const esArchiver = getService('esArchiver');
+  const { setup, tearDown } = getLifecycleMethods(getService);
 
   describe('index detail mb', () => {
     const archive =
@@ -22,11 +23,11 @@ export default function ({ getService }) {
     };
 
     before('load archive', () => {
-      return esArchiver.load(archive);
+      return setup(archive);
     });
 
     after('unload archive', () => {
-      return esArchiver.unload(archive);
+      return tearDown();
     });
 
     it('should summarize index with chart metrics data for the non-advanced view', async () => {

x-pack/test/api_integration/apis/monitoring/elasticsearch/indices_mb.js

@@ -10,10 +10,12 @@ import relocatingShardsFixture from './fixtures/indices_shards_relocating';
 import relocationShardsAllFixture from './fixtures/indices_shards_relocating_all';
 import indicesRedClusterFixture from './fixtures/indices_red_cluster';
 import indicesRedClusterAllFixture from './fixtures/indices_red_cluster_all';
+import { getLifecycleMethods } from '../data_stream';
 
 export default function ({ getService }) {
   const supertest = getService('supertest');
   const esArchiver = getService('esArchiver');
+  const { setup, tearDown } = getLifecycleMethods(getService);
 
   describe('indices mb', () => {
     describe('shard-relocation', () => {
@@ -25,11 +27,11 @@ export default function ({ getService }) {
       };
 
       before('load archive', () => {
-        return esArchiver.load(archive);
+        return setup(archive);
       });
 
       after('unload archive', () => {
-        return esArchiver.unload(archive);
+        return tearDown();
       });
 
       it('should summarize the non-system indices with stats', async () => {

x-pack/test/api_integration/apis/monitoring/elasticsearch/node_detail_advanced_mb.js

@@ -7,10 +7,11 @@
 
 import expect from '@kbn/expect';
 import nodeDetailFixture from './fixtures/node_detail_advanced';
+import { getLifecycleMethods } from '../data_stream';
 
 export default function ({ getService }) {
   const supertest = getService('supertest');
-  const esArchiver = getService('esArchiver');
+  const { setup, tearDown } = getLifecycleMethods(getService);
 
   describe('node detail advanced mb', () => {
     const archive =
@@ -21,11 +22,11 @@ export default function ({ getService }) {
     };
 
     before('load archive', () => {
-      return esArchiver.load(archive);
+      return setup(archive);
     });
 
     after('unload archive', () => {
-      return esArchiver.unload(archive);
+      return tearDown();
     });
 
     it('should summarize node with metrics', async () => {

x-pack/test/api_integration/apis/monitoring/elasticsearch/node_detail_mb.js

@@ -7,10 +7,11 @@
 
 import expect from '@kbn/expect';
 import nodeDetailFixture from './fixtures/node_detail';
+import { getLifecycleMethods } from '../data_stream';
 
 export default function ({ getService }) {
   const supertest = getService('supertest');
-  const esArchiver = getService('esArchiver');
+  const { setup, tearDown } = getLifecycleMethods(getService);
 
   describe('node detail mb', function () {
     // TODO: https://github.com/elastic/stack-monitoring/issues/31
@@ -24,11 +25,11 @@ export default function ({ getService }) {
     };
 
     before('load archive', () => {
-      return esArchiver.load(archive);
+      return setup(archive);
     });
 
     after('unload archive', () => {
-      return esArchiver.unload(archive);
+      return tearDown();
     });
 
     it('should summarize node with metrics', async () => {

x-pack/test/api_integration/apis/monitoring/elasticsearch/nodes_mb.js

@@ -10,10 +10,12 @@ import nodesListingFixtureGreen from './fixtures/nodes_listing_green';
 import nodesListingFixtureRed from './fixtures/nodes_listing_red';
 import nodesListingFixtureCgroup from './fixtures/nodes_listing_cgroup';
 import nodesListingFixturePagination from './fixtures/nodes_listing_pagination';
+import { getLifecycleMethods } from '../data_stream';
 
 export default function ({ getService }) {
   const supertest = getService('supertest');
   const esArchiver = getService('esArchiver');
+  const { setup, tearDown } = getLifecycleMethods(getService);
 
   describe('nodes mb', () => {
     describe('with green platinum cluster', () => {
@@ -29,11 +31,11 @@ export default function ({ getService }) {
       };
 
       before('load clusters archive', () => {
-        return esArchiver.load(archive);
+        return setup(archive);
       });
 
       after('unload clusters archive', () => {
-        return esArchiver.unload(archive);
+        return tearDown();
       });
 
       it('should return data for 2 active nodes', async () => {

x-pack/test/api_integration/apis/monitoring/elasticsearch/overview_mb.js

@@ -10,10 +10,12 @@ import expect from '@kbn/expect';
 import overviewFixtureGreenPlatinum from './fixtures/overview_green_platinum';
 import overviewFixtureRedPlatinum from './fixtures/overview_red_platinum';
 import overviewFixtureShardsRelocating from './fixtures/overview_shards_relocating';
+import { getLifecycleMethods } from '../data_stream';
 
 export default function ({ getService }) {
   const supertest = getService('supertest');
   const esArchiver = getService('esArchiver');
+  const { setup, tearDown } = getLifecycleMethods(getService);
 
   describe('overview mb', () => {
     describe('with green platinum cluster', () => {
@@ -25,11 +27,11 @@ export default function ({ getService }) {
       };
 
       before('load clusters archive', () => {
-        return esArchiver.load(archive);
+        return setup(archive);
       });
 
-      after('unload clusters archive', () => {
-        return esArchiver.unload(archive);
+      after('unload clusters archive', async () => {
+        return tearDown();
       });
 
       it('should summarize elasticsearch with metrics', async () => {