Merge branch '7.x' into backport/7.x/pr-86653

package.json

@@ -349,7 +349,7 @@
     "@babel/traverse": "^7.11.5",
     "@babel/types": "^7.11.0",
     "@cypress/snapshot": "^2.1.7",
-    "@cypress/webpack-preprocessor": "^5.4.11",
+    "@cypress/webpack-preprocessor": "^5.5.0",
     "@elastic/apm-rum": "^5.6.1",
     "@elastic/apm-rum-react": "^1.2.5",
     "@elastic/charts": "24.4.0",
@@ -604,7 +604,7 @@
     "cpy": "^8.1.1",
     "cronstrue": "^1.51.0",
     "css-loader": "^3.4.2",
-    "cypress": "^6.1.0",
+    "cypress": "^6.2.1",
     "cypress-cucumber-preprocessor": "^2.5.2",
     "cypress-multi-reporters": "^1.4.0",
     "d3": "3.5.17",

packages/kbn-es-archiver/src/actions/empty_kibana_index.ts

@@ -20,7 +20,7 @@
 import { Client } from 'elasticsearch';
 import { ToolingLog, KbnClient } from '@kbn/dev-utils';
 
-import { migrateKibanaIndex, deleteKibanaIndices, createStats } from '../lib';
+import { migrateKibanaIndex, createStats, cleanKibanaIndices } from '../lib';
 
 export async function emptyKibanaIndexAction({
   client,
@@ -32,8 +32,9 @@ export async function emptyKibanaIndexAction({
   kbnClient: KbnClient;
 }) {
   const stats = createStats('emptyKibanaIndex', log);
+  const kibanaPluginIds = await kbnClient.plugins.getEnabledIds();
 
-  await deleteKibanaIndices({ client, stats, log });
+  await cleanKibanaIndices({ client, stats, log, kibanaPluginIds });
   await migrateKibanaIndex({ client, kbnClient });
   return stats;
 }

packages/kbn-es-archiver/src/lib/index.ts

@@ -25,6 +25,7 @@ export {
   createGenerateIndexRecordsStream,
   deleteKibanaIndices,
   migrateKibanaIndex,
+  cleanKibanaIndices,
   createDefaultSpace,
 } from './indices';
 

packages/kbn-es-archiver/src/lib/indices/index.ts

@@ -20,4 +20,9 @@
 export { createCreateIndexStream } from './create_index_stream';
 export { createDeleteIndexStream } from './delete_index_stream';
 export { createGenerateIndexRecordsStream } from './generate_index_records_stream';
-export { migrateKibanaIndex, deleteKibanaIndices, createDefaultSpace } from './kibana_index';
+export {
+  migrateKibanaIndex,
+  deleteKibanaIndices,
+  cleanKibanaIndices,
+  createDefaultSpace,
+} from './kibana_index';

packages/kbn-es-archiver/src/lib/indices/kibana_index.ts

@@ -73,6 +73,7 @@ export async function migrateKibanaIndex({
     body: {
       dynamic: true,
     },
+    ignore: [404],
   } as any);
 
   await kbnClient.savedObjects.migrate();

x-pack/plugins/index_management/__jest__/client_integration/index_template_wizard/template_edit.test.tsx

@@ -71,6 +71,10 @@ describe('<TemplateEdit />', () => {
     const templateToEdit = fixtures.getTemplate({
       name: 'index_template_without_mappings',
       indexPatterns: ['indexPattern1'],
+      dataStream: {
+        hidden: true,
+        anyUnknownKey: 'should_be_kept',
+      },
     });
 
     beforeAll(() => {
@@ -85,7 +89,7 @@ describe('<TemplateEdit />', () => {
       testBed.component.update();
     });
 
-    it('allows you to add mappings', async () => {
+    test('allows you to add mappings', async () => {
       const { actions, find } = testBed;
       // Logistics
       await actions.completeStepOne();
@@ -123,11 +127,15 @@ describe('<TemplateEdit />', () => {
       const expected = {
         name: 'index_template_without_mappings',
         indexPatterns: ['indexPattern1'],
+        dataStream: {
+          hidden: true,
+          anyUnknownKey: 'should_be_kept',
+        },
         version,
         _kbnMeta: {
           type: 'default',
           isLegacy: templateToEdit._kbnMeta.isLegacy,
-          hasDatastream: false,
+          hasDatastream: true,
         },
       };
 
@@ -152,6 +160,47 @@ describe('<TemplateEdit />', () => {
 
       expect(exists('requestTab')).toBe(true);
     });
+
+    test('should keep data stream configuration', async () => {
+      const { actions } = testBed;
+      // Logistics
+      await actions.completeStepOne({
+        name: 'test',
+        indexPatterns: ['myPattern*'],
+        version: 1,
+      });
+      // Component templates
+      await actions.completeStepTwo();
+      // Index settings
+      await actions.completeStepThree();
+      // Mappings
+      await actions.completeStepFour();
+      // Aliases
+      await actions.completeStepFive();
+
+      await act(async () => {
+        actions.clickNextButton();
+      });
+
+      const latestRequest = server.requests[server.requests.length - 1];
+
+      const expected = {
+        name: 'test',
+        indexPatterns: ['myPattern*'],
+        dataStream: {
+          hidden: true,
+          anyUnknownKey: 'should_be_kept',
+        },
+        version: 1,
+        _kbnMeta: {
+          type: 'default',
+          isLegacy: false,
+          hasDatastream: true,
+        },
+      };
+
+      expect(JSON.parse(JSON.parse(latestRequest.requestBody).body)).toEqual(expected);
+    });
   });
 
   describe('with mappings', () => {

x-pack/plugins/index_management/common/types/templates.ts

@@ -46,7 +46,11 @@ export interface TemplateDeserialized {
     name: string;
   };
   _meta?: { [key: string]: any }; // Composable template only
-  dataStream?: {}; // Composable template only
+  // Composable template only
+  dataStream?: {
+    hidden?: boolean;
+    [key: string]: any;
+  };
   _kbnMeta: {
     type: TemplateType;
     hasDatastream: boolean;

x-pack/plugins/index_management/public/application/components/template_form/steps/step_logistics.tsx

@@ -55,7 +55,7 @@ function getFieldsMeta(esDocsBase: string) {
       ),
       testSubject: 'indexPatternsField',
     },
-    dataStream: {
+    createDataStream: {
       title: i18n.translate('xpack.idxMgmt.templateForm.stepLogistics.dataStreamTitle', {
         defaultMessage: 'Data stream',
       }),
@@ -119,6 +119,7 @@ interface LogisticsForm {
 
 interface LogisticsFormInternal extends LogisticsForm {
   addMeta: boolean;
+  doCreateDataStream: boolean;
 }
 
 interface Props {
@@ -132,12 +133,16 @@ function formDeserializer(formData: LogisticsForm): LogisticsFormInternal {
   return {
     ...formData,
     addMeta: Boolean(formData._meta && Object.keys(formData._meta).length),
+    doCreateDataStream: Boolean(formData.dataStream),
   };
 }
 
-function formSerializer(formData: LogisticsFormInternal): LogisticsForm {
-  const { addMeta, ...rest } = formData;
-  return rest;
+function getformSerializer(initialTemplateData: LogisticsForm = {}) {
+  return (formData: LogisticsFormInternal): LogisticsForm => {
+    const { addMeta, doCreateDataStream, ...rest } = formData;
+    const dataStream = doCreateDataStream ? initialTemplateData.dataStream ?? {} : undefined;
+    return { ...rest, dataStream };
+  };
 }
 
 export const StepLogistics: React.FunctionComponent<Props> = React.memo(
@@ -146,7 +151,7 @@ export const StepLogistics: React.FunctionComponent<Props> = React.memo(
       schema: schemas.logistics,
       defaultValue,
       options: { stripEmptyFields: false },
-      serializer: formSerializer,
+      serializer: getformSerializer(defaultValue),
       deserializer: formDeserializer,
     });
     const {
@@ -178,7 +183,7 @@ export const StepLogistics: React.FunctionComponent<Props> = React.memo(
       });
     }, [onChange, isFormValid, validate, getFormData]);
 
-    const { name, indexPatterns, dataStream, order, priority, version } = getFieldsMeta(
+    const { name, indexPatterns, createDataStream, order, priority, version } = getFieldsMeta(
       documentationService.getEsDocsBase()
     );
 
@@ -245,10 +250,10 @@ export const StepLogistics: React.FunctionComponent<Props> = React.memo(
 
           {/* Create data stream */}
           {isLegacy !== true && (
-            <FormRow title={dataStream.title} description={dataStream.description}>
+            <FormRow title={createDataStream.title} description={createDataStream.description}>
               <UseField
-                path="dataStream"
-                componentProps={{ 'data-test-subj': dataStream.testSubject }}
+                path="doCreateDataStream"
+                componentProps={{ 'data-test-subj': createDataStream.testSubject }}
               />
             </FormRow>
           )}

x-pack/plugins/index_management/public/application/components/template_form/template_form_schemas.tsx

@@ -129,31 +129,12 @@ export const schemas: Record<string, FormSchema> = {
         },
       ],
     },
-    dataStream: {
+    doCreateDataStream: {
       type: FIELD_TYPES.TOGGLE,
       label: i18n.translate('xpack.idxMgmt.templateForm.stepLogistics.datastreamLabel', {
         defaultMessage: 'Create data stream',
       }),
       defaultValue: false,
-      serializer: (value) => {
-        if (value === true) {
-          // For now, ES expects an empty object when defining a data stream
-          // https://github.com/elastic/elasticsearch/pull/59317
-          return {};
-        }
-      },
-      deserializer: (value) => {
-        if (typeof value === 'boolean') {
-          return value;
-        }
-
-        /**
-         * For now, it is enough to have a "data_stream" declared on the index template
-         * to assume that the template creates a data stream. In the future, this condition
-         * might change
-         */
-        return value !== undefined;
-      },
     },
     order: {
       type: FIELD_TYPES.NUMBER,

x-pack/plugins/index_management/server/routes/api/templates/validate_schemas.ts

@@ -20,7 +20,14 @@ export const templateSchema = schema.object({
     })
   ),
   composedOf: schema.maybe(schema.arrayOf(schema.string())),
-  dataStream: schema.maybe(schema.object({}, { unknowns: 'allow' })),
+  dataStream: schema.maybe(
+    schema.object(
+      {
+        hidden: schema.maybe(schema.boolean()),
+      },
+      { unknowns: 'allow' }
+    )
+  ),
   _meta: schema.maybe(schema.object({}, { unknowns: 'allow' })),
   ilmPolicy: schema.maybe(
     schema.object({

x-pack/plugins/index_management/test/fixtures/template.ts

@@ -53,6 +53,7 @@ export const getTemplate = ({
   order = getRandomNumber(),
   indexPatterns = [],
   template: { settings, aliases, mappings } = {},
+  dataStream,
   hasDatastream = false,
   isLegacy = false,
   type = 'default',
@@ -73,12 +74,13 @@ export const getTemplate = ({
       mappings,
       settings,
     },
+    dataStream,
     hasSettings: objHasProperties(settings),
     hasMappings: objHasProperties(mappings),
     hasAliases: objHasProperties(aliases),
     _kbnMeta: {
       type,
-      hasDatastream,
+      hasDatastream: dataStream !== undefined ? true : hasDatastream,
       isLegacy,
     },
   };

x-pack/plugins/security_solution/cypress/cypress.json

@@ -2,6 +2,7 @@
   "baseUrl": "http://localhost:5601",
   "defaultCommandTimeout": 60000,
   "execTimeout": 120000,
+  "pageLoadTimeout": 120000,
   "nodeVersion": "system",
   "retries": {
     "runMode": 2

x-pack/plugins/security_solution/cypress/integration/alerts_detection_callouts_readonly.spec.ts

@@ -39,7 +39,7 @@ describe('Detections > Callouts indicating read-only access to resources', () =>
   const RULES_CALLOUT = 'read-only-access-to-rules';
 
   before(() => {
-    // First, we have to open the app on behalf of a priviledged user in order to initialize it.
+    // First, we have to open the app on behalf of a privileged user in order to initialize it.
     // Otherwise the app will be disabled and show a "welcome"-like page.
     cleanKibana();
     loginAndWaitForPageWithoutDateRange(DETECTIONS_URL, ROLES.platform_engineer);

x-pack/plugins/security_solution/cypress/integration/alerts_detection_rules.spec.ts

@@ -31,14 +31,13 @@ import { loginAndWaitForPageWithoutDateRange } from '../tasks/login';
 import { DEFAULT_RULE_REFRESH_INTERVAL_VALUE } from '../../common/constants';
 
 import { DETECTIONS_URL } from '../urls/navigation';
-import { createCustomRule, removeSignalsIndex } from '../tasks/api_calls/rules';
+import { createCustomRule } from '../tasks/api_calls/rules';
 import { cleanKibana } from '../tasks/common';
 import { existingRule, newOverrideRule, newRule, newThresholdRule } from '../objects/rule';
 
 describe('Alerts detection rules', () => {
   beforeEach(() => {
     cleanKibana();
-    removeSignalsIndex();
     loginAndWaitForPageWithoutDateRange(DETECTIONS_URL);
     waitForAlertsPanelToBeLoaded();
     waitForAlertsIndexToBeCreated();