Support for reindexing APM indices (#29845)

Signed-off-by: Tyler Smalley <[email protected]>

src/legacy/core_plugins/apm_oss/index.d.ts

@@ -0,0 +1,22 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export interface ApmOssPlugin {
+  indexPatterns: string[];
+}

src/legacy/core_plugins/apm_oss/index.js

@@ -17,6 +17,8 @@
  * under the License.
  */
 
+import _ from 'lodash';
+
 export default function apmOss(kibana) {
   return new kibana.Plugin({
     id: 'apm_oss',
@@ -30,12 +32,24 @@ export default function apmOss(kibana) {
         indexPattern: Joi.string().default('apm-*'),
 
         // ES Indices
+        sourcemapIndices: Joi.string().default('apm-*'),
         errorIndices: Joi.string().default('apm-*'),
-        onboardingIndices: Joi.string().default('apm-*'),
-        spanIndices: Joi.string().default('apm-*'),
         transactionIndices: Joi.string().default('apm-*'),
+        spanIndices: Joi.string().default('apm-*'),
         metricsIndices: Joi.string().default('apm-*'),
+        onboardingIndices: Joi.string().default('apm-*'),
       }).default();
     },
+
+    init(server) {
+      server.expose('indexPatterns', _.uniq([
+        'sourcemapIndices',
+        'errorIndices',
+        'transactionIndices',
+        'spanIndices',
+        'metricsIndices',
+        'onboardingIndices'
+      ].map(type => server.config().get(`apm_oss.${type}`))));
+    }
   });
 }

src/legacy/core_plugins/elasticsearch/index.d.ts

@@ -202,13 +202,15 @@ export interface DeprecationInfo {
   details?: string;
 }
 
+export interface IndexSettingsDeprecationInfo {
+  [indexName: string]: DeprecationInfo[];
+}
+
 export interface DeprecationAPIResponse {
   cluster_settings: DeprecationInfo[];
   ml_settings: DeprecationInfo[];
   node_settings: DeprecationInfo[];
-  index_settings: {
-    [indexName: string]: DeprecationInfo[];
-  };
+  index_settings: IndexSettingsDeprecationInfo;
 }
 
 export interface CallClusterOptions {

src/server/kbn_server.d.ts

@@ -19,7 +19,9 @@
 
 import { Server } from 'hapi';
 
+import { ApmOssPlugin } from '../legacy/core_plugins/apm_oss';
 import { CallClusterWithRequest, ElasticsearchPlugin } from '../legacy/core_plugins/elasticsearch';
+
 import { IndexPatternsServiceFactory } from './index_patterns';
 import { SavedObjectsClient, SavedObjectsService } from './saved_objects';
 
@@ -33,6 +35,7 @@ declare module 'hapi' {
     elasticsearch: ElasticsearchPlugin;
     kibana: any;
     spaces: any;
+    apm_oss: ApmOssPlugin;
     // add new plugin types here
   }
 

x-pack/plugins/upgrade_assistant/common/types.ts

@@ -53,6 +53,7 @@ export enum ReindexWarning {
   booleanFields = 1,
 
   // 7.0 -> 8.0 warnings
+  apmReindex,
 }
 
 export enum IndexGroup {

x-pack/plugins/upgrade_assistant/index.ts

@@ -3,8 +3,8 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-import { Server } from 'hapi';
 import Joi from 'joi';
+import { Legacy } from 'kibana';
 import { resolve } from 'path';
 import mappings from './mappings.json';
 import { initServer } from './server';
@@ -35,7 +35,7 @@ export function upgradeAssistant(kibana: any) {
       }).default();
     },
 
-    init(server: Server) {
+    init(server: Legacy.Server) {
       // Add server routes and initialize the plugin here
       initServer(server);
     },

x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/index_table.tsx

@@ -135,15 +135,15 @@ export class IndexDeprecationTableUI extends React.Component<
     // NOTE: this naive implementation assumes all indices in the table are
     // should show the reindex button. This should work for known usecases.
     const { indices } = this.props;
-    if (!indices.find(i => i.reindex)) {
+    if (!indices.find(i => i.reindex === true)) {
       return null;
     }
 
     return {
       actions: [
         {
           render(indexDep: IndexDeprecationDetails) {
-            return <ReindexButton indexName={indexDep.index} />;
+            return <ReindexButton indexName={indexDep.index!} />;
           },
         },
       ],

x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/list.tsx

@@ -10,13 +10,10 @@ import { DeprecationInfo } from 'src/legacy/core_plugins/elasticsearch';
 import { EnrichedDeprecationInfo } from '../../../../../server/lib/es_migration_apis';
 import { GroupByOption } from '../../../types';
 
-import { CURRENT_MAJOR_VERSION } from 'x-pack/plugins/upgrade_assistant/common/version';
 import { COLOR_MAP, LEVEL_MAP } from '../constants';
 import { DeprecationCell } from './cell';
 import { IndexDeprecationDetails, IndexDeprecationTable } from './index_table';
 
-const OLD_INDEX_MESSAGE = `Index created before ${CURRENT_MAJOR_VERSION}.0`;
-
 const sortByLevelDesc = (a: DeprecationInfo, b: DeprecationInfo) => {
   return -1 * (LEVEL_MAP[a.level] - LEVEL_MAP[b.level]);
 };
@@ -37,7 +34,7 @@ const MessageDeprecation: StatelessComponent<{ deprecation: EnrichedDeprecationI
     <DeprecationCell
       headline={deprecation.message}
       healthColor={COLOR_MAP[deprecation.level]}
-      reindexIndexName={deprecation.message === OLD_INDEX_MESSAGE ? deprecation.index! : undefined}
+      reindexIndexName={deprecation.reindex ? deprecation.index! : undefined}
       docUrl={deprecation.url}
       items={items}
     />
@@ -91,9 +88,8 @@ export const DeprecationList: StatelessComponent<{
     const indices = deprecations.map(dep => ({
       index: dep.index!,
       details: dep.details,
-      reindex: dep.message === OLD_INDEX_MESSAGE,
+      reindex: dep.reindex === true,
     }));
-
     return <IndexDeprecation indices={indices} deprecation={deprecations[0]} />;
   } else if (currentGroupBy === GroupByOption.index) {
     return (

x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/flyout/__snapshots__/warning_step.test.tsx.snap

@@ -108,8 +108,7 @@ exports[`WarningsFlyoutStep renders 1`] = `
         <EuiCode>
           1
         </EuiCode>
-        ), reindexing converts these fields to
-         
+        ), reindexing converts these fields to 
         <EuiCode>
           true
         </EuiCode>
@@ -129,6 +128,9 @@ exports[`WarningsFlyoutStep renders 1`] = `
         </EuiLink>
       </p>
     </EuiText>
+    <EuiSpacer
+      size="l"
+    />
   </EuiFlyoutBody>
   <EuiFlyoutFooter>
     <EuiFlexGroup

x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/flyout/warnings_step.tsx

@@ -79,61 +79,93 @@ export class WarningsFlyoutStep extends React.Component<
           <EuiSpacer />
 
           {warnings.includes(ReindexWarning.allField) && (
-            <EuiText>
-              <EuiCheckbox
-                id={idForWarning(ReindexWarning.allField)}
-                label={
-                  <strong>
-                    <EuiCode>_all</EuiCode> field will be removed
-                  </strong>
-                }
-                checked={checkedIds[idForWarning(ReindexWarning.allField)]}
-                onChange={this.onChange}
-              />
-              <p className="upgWarningsStep__warningDescription">
-                The <EuiCode>_all</EuiCode> meta field is no longer supported in 7.0. Reindexing
-                removes the <EuiCode>_all</EuiCode> field in the new index. Ensure that no
-                application code or scripts reply on this field.
-                <br />
-                <EuiLink
-                  href="https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_the_literal__all_literal_meta_field_is_now_disabled_by_default"
-                  target="_blank"
-                >
-                  Documentation
-                </EuiLink>
-              </p>
-            </EuiText>
+            <Fragment>
+              <EuiText>
+                <EuiCheckbox
+                  id={idForWarning(ReindexWarning.allField)}
+                  label={
+                    <strong>
+                      <EuiCode>_all</EuiCode> field will be removed
+                    </strong>
+                  }
+                  checked={checkedIds[idForWarning(ReindexWarning.allField)]}
+                  onChange={this.onChange}
+                />
+                <p className="upgWarningsStep__warningDescription">
+                  The <EuiCode>_all</EuiCode> meta field is no longer supported in 7.0. Reindexing
+                  removes the <EuiCode>_all</EuiCode> field in the new index. Ensure that no
+                  application code or scripts reply on this field.
+                  <br />
+                  <EuiLink
+                    href="https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_the_literal__all_literal_meta_field_is_now_disabled_by_default"
+                    target="_blank"
+                  >
+                    Documentation
+                  </EuiLink>
+                </p>
+              </EuiText>
+
+              <EuiSpacer />
+            </Fragment>
           )}
 
-          <EuiSpacer />
+          {warnings.includes(ReindexWarning.apmReindex) && (
+            <Fragment>
+              <EuiText>
+                <EuiCheckbox
+                  id={idForWarning(ReindexWarning.apmReindex)}
+                  label={<strong>This index will be converted to ECS format</strong>}
+                  checked={checkedIds[idForWarning(ReindexWarning.apmReindex)]}
+                  onChange={this.onChange}
+                />
+                <p className="upgWarningsStep__warningDescription">
+                  Starting in version 7.0.0, APM data will be represented in the Elastic Common
+                  Schema. Historical APM data will not visible until it's reindexed.
+                  <br />
+                  <EuiLink
+                    href="https://www.elastic.co/guide/en/apm/get-started/master/apm-release-notes.html"
+                    target="_blank"
+                  >
+                    Documentation
+                  </EuiLink>
+                </p>
+              </EuiText>
+
+              <EuiSpacer />
+            </Fragment>
+          )}
 
           {warnings.includes(ReindexWarning.booleanFields) && (
-            <EuiText>
-              <EuiCheckbox
-                id={idForWarning(ReindexWarning.booleanFields)}
-                label={
-                  <strong>
-                    Boolean data in <EuiCode>_source</EuiCode> might change
-                  </strong>
-                }
-                checked={checkedIds[idForWarning(ReindexWarning.booleanFields)]}
-                onChange={this.onChange}
-              />
-              <p className="upgWarningsStep__warningDescription">
-                If a documents contain a boolean field that is neither <EuiCode>true</EuiCode> or{' '}
-                <EuiCode>false</EuiCode> (for example, <EuiCode>"yes"</EuiCode>,{' '}
-                <EuiCode>"on"</EuiCode>, <EuiCode>1</EuiCode>), reindexing converts these fields to{' '}
-                <EuiCode>true</EuiCode> or <EuiCode>false</EuiCode>. Ensure that no application code
-                or scripts rely on boolean fields in the deprecated format.
-                <br />
-                <EuiLink
-                  href="https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_coercion_of_boolean_fields"
-                  target="_blank"
-                >
-                  Documentation
-                </EuiLink>
-              </p>
-            </EuiText>
+            <Fragment>
+              <EuiText>
+                <EuiCheckbox
+                  id={idForWarning(ReindexWarning.booleanFields)}
+                  label={
+                    <strong>
+                      Boolean data in <EuiCode>_source</EuiCode> might change
+                    </strong>
+                  }
+                  checked={checkedIds[idForWarning(ReindexWarning.booleanFields)]}
+                  onChange={this.onChange}
+                />
+                <p className="upgWarningsStep__warningDescription">
+                  If a documents contain a boolean field that is neither <EuiCode>true</EuiCode> or{' '}
+                  <EuiCode>false</EuiCode> (for example, <EuiCode>"yes"</EuiCode>,{' '}
+                  <EuiCode>"on"</EuiCode>, <EuiCode>1</EuiCode>), reindexing converts these fields
+                  to <EuiCode>true</EuiCode> or <EuiCode>false</EuiCode>. Ensure that no application
+                  code or scripts rely on boolean fields in the deprecated format.
+                  <br />
+                  <EuiLink
+                    href="https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_coercion_of_boolean_fields"
+                    target="_blank"
+                  >
+                    Documentation
+                  </EuiLink>
+                </p>
+              </EuiText>
+
+              <EuiSpacer />
+            </Fragment>
           )}
         </EuiFlyoutBody>
         <EuiFlyoutFooter>

x-pack/plugins/upgrade_assistant/server/lib/__snapshots__/es_migration_apis.test.ts.snap

@@ -34,41 +34,47 @@ Object {
       "index": ".monitoring-es-6-2018.11.07",
       "level": "warning",
       "message": "Coercion of boolean fields",
+      "reindex": false,
       "url": "https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_coercion_of_boolean_fields",
     },
     Object {
       "details": "[[type: tweet, field: liked]]",
       "index": "twitter",
       "level": "warning",
       "message": "Coercion of boolean fields",
+      "reindex": false,
       "url": "https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_coercion_of_boolean_fields",
     },
     Object {
       "details": "[[type: index-pattern, field: notExpandable], [type: config, field: xPackMonitoring:allowReport], [type: config, field: xPackMonitoring:showBanner], [type: dashboard, field: pause], [type: dashboard, field: timeRestore]]",
       "index": ".kibana",
       "level": "warning",
       "message": "Coercion of boolean fields",
+      "reindex": false,
       "url": "https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_coercion_of_boolean_fields",
     },
     Object {
       "details": "[[type: doc, field: notify], [type: doc, field: created], [type: doc, field: attach_payload], [type: doc, field: met]]",
       "index": ".watcher-history-6-2018.11.07",
       "level": "warning",
       "message": "Coercion of boolean fields",
+      "reindex": false,
       "url": "https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_coercion_of_boolean_fields",
     },
     Object {
       "details": "[[type: doc, field: snapshot]]",
       "index": ".monitoring-kibana-6-2018.11.07",
       "level": "warning",
       "message": "Coercion of boolean fields",
+      "reindex": false,
       "url": "https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_coercion_of_boolean_fields",
     },
     Object {
       "details": "[[type: tweet, field: liked]]",
       "index": "twitter2",
       "level": "warning",
       "message": "Coercion of boolean fields",
+      "reindex": false,
       "url": "https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_coercion_of_boolean_fields",
     },
   ],