[Response Ops][Alerting] Adding ignore_malformed to .alerts-* ind…

…ex template settings (elastic#163414) Resolves elastic#161465 ## Summary Adds `ignore_malformed: true` to alerts index template settings. This ignores malformed content globally across all allowed mapping types. For existing alerts as data indices, the new setting is not applied directly to the existing concrete indices but will be applied whenever the alias rolls over and a new concrete index is created. ## Verify - Verify that after upgrading alerts indices created in an older version to this branch, alerts continue to be written and read as expected. --------- Co-authored-by: kibanamachine <[email protected]> Co-authored-by: Vitalii Dmyterko <[email protected]>
bryce-b · Aug 9, 2023 · e3db0a4 · e3db0a4
1 parent 39dc223
commit e3db0a4
Show file tree

Hide file tree

Showing 5 changed files with 11 additions and 6 deletions.
diff --git a/x-pack/plugins/alerting/server/alerts_service/alerts_service.test.ts b/x-pack/plugins/alerting/server/alerts_service/alerts_service.test.ts
@@ -114,6 +114,7 @@ const getIndexTemplatePutBody = (opts?: GetIndexTemplatePutBodyOpts) => {
             name: '.alerts-ilm-policy',
             rollover_alias: `.alerts-${context ? context : 'test'}.alerts-${namespace}`,
           },
+          'index.mapping.ignore_malformed': true,
           'index.mapping.total_fields.limit': 2500,
         },
         mappings: {
@@ -640,6 +641,7 @@ describe('Alerts Service', () => {
                 name: '.alerts-ilm-policy',
                 rollover_alias: `.alerts-empty.alerts-default`,
               },
+              'index.mapping.ignore_malformed': true,
               'index.mapping.total_fields.limit': 2500,
             },
             mappings: {

diff --git a/x-pack/plugins/alerting/server/alerts_service/lib/create_or_update_index_template.test.ts b/x-pack/plugins/alerting/server/alerts_service/lib/create_or_update_index_template.test.ts
@@ -42,6 +42,7 @@ const IndexTemplate = (namespace: string = 'default') => ({
           name: 'test-ilm-policy',
           rollover_alias: `.alerts-test.alerts-${namespace}`,
         },
+        'index.mapping.ignore_malformed': true,
         'index.mapping.total_fields.limit': 2500,
       },
     },

diff --git a/x-pack/plugins/alerting/server/alerts_service/lib/create_or_update_index_template.ts b/x-pack/plugins/alerting/server/alerts_service/lib/create_or_update_index_template.ts
@@ -54,6 +54,7 @@ export const getIndexTemplate = ({
             rollover_alias: indexPatterns.alias,
           },
           'index.mapping.total_fields.limit': totalFieldsLimit,
+          'index.mapping.ignore_malformed': true,
         },
         mappings: {
           dynamic: false,

diff --git a/...ing_api_integration/spaces_only/tests/alerting/group4/alerts_as_data/install_resources.ts b/...ing_api_integration/spaces_only/tests/alerting/group4/alerts_as_data/install_resources.ts
@@ -163,6 +163,7 @@ export default function createAlertsAsDataInstallResourcesTest({ getService }: F
             rollover_alias: '.alerts-test.patternfiring.alerts-default',
           },
           mapping: {
+            ignore_malformed: 'true',
             total_fields: {
               limit: '2500',
             },
@@ -196,6 +197,7 @@ export default function createAlertsAsDataInstallResourcesTest({ getService }: F
       });
 
       expect(contextIndex[indexName].settings?.index?.mapping).to.eql({
+        ignore_malformed: 'true',
         total_fields: {
           limit: '2500',
         },

diff --git a/...tection_engine_api_integration/security_and_spaces/rule_execution_logic/non_ecs_fields.ts b/...tection_engine_api_integration/security_and_spaces/rule_execution_logic/non_ecs_fields.ts
@@ -56,7 +56,6 @@ export default ({ getService }: FtrProviderContext) => {
     };
   };
 
-  // FAILING ES PROMOTION: https://github.com/elastic/kibana/issues/154277
   describe('Non ECS fields in alert document source', () => {
     before(async () => {
       await esArchiver.load(
@@ -259,6 +258,7 @@ export default ({ getService }: FtrProviderContext) => {
 
     // we don't validate it because geo_point is very complex type with many various representations: array, different object, string with few valid patterns
     // more on geo_point type https://www.elastic.co/guide/en/elasticsearch/reference/current/geo-point.html
+    // since .alerts-* indices allow _ignore_malformed option, alert will be indexed for this document
     it('should fail creating alert when ECS field mapping is geo_point', async () => {
       const document = {
         client: {
@@ -269,12 +269,11 @@ export default ({ getService }: FtrProviderContext) => {
         },
       };
 
-      const { errors } = await indexAndCreatePreviewAlert(document);
+      const { errors, alertSource } = await indexAndCreatePreviewAlert(document);
 
-      expect(errors[0]).toContain('Bulk Indexing of signals failed');
-      expect(errors[0]).toContain(
-        'failed to parse field [client.geo.location] of type [geo_point]'
-      );
+      expect(errors).toEqual([]);
+
+      expect(alertSource).toHaveProperty('client.geo.location', 'test test');
     });
 
     it('should strip invalid boolean values and left valid ones', async () => {