[Security Solution][Detection Engine] fixes ES|QL ECS multifiefields …

…issue (elastic#167769)

## Summary

- fixes elastic/security-team#7741 by
replacing `ecsMap` from hardcoded `@kbn/rule-registry-plugin` to actual
mapping for alerts indices from `@kbn/alerts-as-data-utils`
- when converting ES|QL row table results to object, `null` values
skipped, since its results consists of all existing mappings in searched
indices, if fields in query are not filtered

### Checklist

Delete any items that are not applicable to this PR.


- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql.ts

@@ -100,7 +100,7 @@ export const esqlExecutor = async ({
       });
 
       const esqlSearchDuration = makeFloatString(performance.now() - esqlSignalSearchStart);
-      result.searchAfterTimes = [esqlSearchDuration];
+      result.searchAfterTimes.push(esqlSearchDuration);
 
       ruleExecutionLogger.debug(`ES|QL query request took: ${esqlSearchDuration}ms`);
 

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/utils/row_to_document.test.ts

@@ -19,7 +19,6 @@ describe('rowToDocument', () => {
     const row = ['abcd', null, '8.8.1', 'packetbeat'];
     expect(rowToDocument(columns, row)).toEqual({
       _id: 'abcd',
-      'agent.name': null,
       'agent.version': '8.8.1',
       'agent.type': 'packetbeat',
     });

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/utils/row_to_document.ts

@@ -18,8 +18,10 @@ export const rowToDocument = (
   row: EsqlResultRow
 ): Record<string, string | null> => {
   return columns.reduce<Record<string, string | null>>((acc, column, i) => {
-    acc[column.name] = row[i];
-
+    // skips nulls, as ES|QL return null for each existing mapping field
+    if (row[i] !== null) {
+      acc[column.name] = row[i];
+    }
     return acc;
   }, {});
 };

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/strip_non_ecs_fields.test.ts

@@ -71,6 +71,27 @@ describe('stripNonEcsFields', () => {
     ]);
   });
 
+  // https://github.com/elastic/sdh-security-team/issues/736
+  describe('fields that exists in the alerts mapping but not in local ECS(ruleRegistry) definition', () => {
+    it('should strip object type "device" field if it is supplied as a keyword', () => {
+      const { result, removed } = stripNonEcsFields({
+        device: 'test',
+        message: 'test message',
+      });
+
+      expect(result).toEqual({
+        message: 'test message',
+      });
+
+      expect(removed).toEqual([
+        {
+          key: 'device',
+          value: 'test',
+        },
+      ]);
+    });
+  });
+
   describe('array fields', () => {
     it('should not strip arrays of objects when an object is expected', () => {
       const { result, removed } = stripNonEcsFields({

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/factories/utils/strip_non_ecs_fields.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { ecsFieldMap } from '@kbn/rule-registry-plugin/common/assets/field_maps/ecs_field_map';
+import { ecsFieldMap } from '@kbn/alerts-as-data-utils';
 
 import { isPlainObject, cloneDeep, isArray } from 'lodash';
 

x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/esql.ts

@@ -116,9 +116,6 @@ export default ({ getService }: FtrProviderContext) => {
         'kibana.space_ids': ['default'],
         'kibana.alert.rule.tags': [],
         'agent.name': 'test-1',
-        'agent.type': null,
-        'agent.version': null,
-        'host.name': null,
         id,
         'event.kind': 'signal',
         'kibana.alert.original_time': expect.any(String),
@@ -155,8 +152,6 @@ export default ({ getService }: FtrProviderContext) => {
         'kibana.alert.workflow_tags': [],
         'kibana.alert.rule.risk_score': 55,
         'kibana.alert.rule.severity': 'high',
-        'kibana.alert.original_event.created': null,
-        'kibana.alert.original_event.ingested': null,
       });
     });
 
@@ -829,5 +824,163 @@ export default ({ getService }: FtrProviderContext) => {
         expect(previewAlerts[0]._source).toHaveProperty('host.risk.calculated_score_norm', 1);
       });
     });
+
+    describe('ECS fields validation', () => {
+      it('creates alert if ECS field has multifields', async () => {
+        const id = uuidv4();
+        const interval: [string, string] = ['2020-10-28T06:00:00.000Z', '2020-10-28T06:10:00.000Z'];
+        const doc1 = { agent: { name: 'test-1' }, 'observer.os.full': 'full test os' };
+        const doc2 = { agent: { name: 'test-2' } };
+
+        const rule: EsqlRuleCreateProps = {
+          ...getCreateEsqlRulesSchemaMock('rule-1', true),
+          query: `from ecs_compliant [metadata _id] ${internalIdPipe(
+            id
+          )} | where agent.name=="test-1"`,
+          from: 'now-1h',
+          interval: '1h',
+        };
+
+        await indexEnhancedDocuments({
+          documents: [doc1, doc2],
+          interval,
+          id,
+        });
+
+        const { previewId } = await previewRule({
+          supertest,
+          rule,
+          timeframeEnd: new Date('2020-10-28T06:30:00.000Z'),
+        });
+
+        const previewAlerts = await getPreviewAlerts({
+          es,
+          previewId,
+          size: 10,
+        });
+
+        expect(previewAlerts.length).toBe(1);
+        expect(previewAlerts[0]._source).toHaveProperty(['observer.os.full'], 'full test os');
+        // *.text is multifield define in mappings for observer.os.full
+        expect(previewAlerts[0]._source).not.toHaveProperty(['observer.os.full.text']);
+      });
+      // https://github.com/elastic/security-team/issues/7741
+      it('creates alert if ECS field has multifields and is not in ruleRegistry ECS map', async () => {
+        const id = uuidv4();
+        const interval: [string, string] = ['2020-10-28T06:00:00.000Z', '2020-10-28T06:10:00.000Z'];
+        const doc1 = {
+          agent: { name: 'test-1' },
+          // this field is mapped in alerts index, but not in ruleRegistry ECS map
+          'process.entry_leader.name': 'test_process_name',
+        };
+        const doc2 = { agent: { name: 'test-2' } };
+
+        const rule: EsqlRuleCreateProps = {
+          ...getCreateEsqlRulesSchemaMock('rule-1', true),
+          query: `from ecs_compliant [metadata _id] ${internalIdPipe(
+            id
+          )} | where agent.name=="test-1"`,
+          from: 'now-1h',
+          interval: '1h',
+        };
+
+        await indexEnhancedDocuments({
+          documents: [doc1, doc2],
+          interval,
+          id,
+        });
+
+        const { previewId } = await previewRule({
+          supertest,
+          rule,
+          timeframeEnd: new Date('2020-10-28T06:30:00.000Z'),
+        });
+
+        const previewAlerts = await getPreviewAlerts({
+          es,
+          previewId,
+          size: 10,
+        });
+
+        expect(previewAlerts.length).toBe(1);
+        expect(previewAlerts[0]._source).toHaveProperty(
+          ['process.entry_leader.name'],
+          'test_process_name'
+        );
+        expect(previewAlerts[0]._source).not.toHaveProperty(['process.entry_leader.name.text']);
+        expect(previewAlerts[0]._source).not.toHaveProperty(['process.entry_leader.name.caseless']);
+      });
+
+      describe('non-ecs', () => {
+        before(async () => {
+          await esArchiver.load(
+            'x-pack/test/functional/es_archives/security_solution/ecs_non_compliant'
+          );
+        });
+
+        after(async () => {
+          await esArchiver.unload(
+            'x-pack/test/functional/es_archives/security_solution/ecs_non_compliant'
+          );
+        });
+
+        const { indexEnhancedDocuments: indexEnhancedDocumentsToNonEcs } = dataGeneratorFactory({
+          es,
+          index: 'ecs_non_compliant',
+          log,
+        });
+
+        it('creates alert if non ECS field has multifields', async () => {
+          const id = uuidv4();
+          const interval: [string, string] = [
+            '2020-10-28T06:00:00.000Z',
+            '2020-10-28T06:10:00.000Z',
+          ];
+          const doc1 = {
+            'random.entry_leader.name': 'random non-ecs field',
+          };
+
+          const rule: EsqlRuleCreateProps = {
+            ...getCreateEsqlRulesSchemaMock('rule-1', true),
+            query: `from ecs_non_compliant [metadata _id] ${internalIdPipe(id)}`,
+            from: 'now-1h',
+            interval: '1h',
+          };
+
+          await indexEnhancedDocumentsToNonEcs({
+            documents: [doc1],
+            interval,
+            id,
+          });
+
+          const { previewId } = await previewRule({
+            supertest,
+            rule,
+            timeframeEnd: new Date('2020-10-28T06:30:00.000Z'),
+          });
+
+          const previewAlerts = await getPreviewAlerts({
+            es,
+            previewId,
+            size: 10,
+          });
+
+          expect(previewAlerts.length).toBe(1);
+          // all multifields have been indexed, which is expected, seen we don't know original mappings
+          expect(previewAlerts[0]._source).toHaveProperty(
+            ['random.entry_leader.name'],
+            'random non-ecs field'
+          );
+          expect(previewAlerts[0]._source).toHaveProperty(
+            ['random.entry_leader.name.text'],
+            'random non-ecs field'
+          );
+          expect(previewAlerts[0]._source).toHaveProperty(
+            ['random.entry_leader.name.caseless'],
+            'random non-ecs field'
+          );
+        });
+      });
+    });
   });
 };

x-pack/test/functional/es_archives/security_solution/ecs_compliant/mappings.json

@@ -23,6 +23,45 @@
             }
           }
         },
+        "observer": {
+          "properties": {
+            "os": {
+              "properties": {
+                "full": {
+                  "type": "keyword",
+                  "ignore_above": 1024,
+                  "fields": {
+                    "text": {
+                      "type": "match_only_text"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        },
+        "process": {
+          "properties": {
+            "entry_leader": {
+              "properties": {
+                "name": {
+                  "type": "keyword",
+                  "ignore_above": 1024,
+                  "fields": {
+                    "text": {
+                      "type": "match_only_text"
+                    },
+                    "caseless": {
+                      "type": "keyword",
+                      "ignore_above": 1024,
+                      "normalizer": "lowercase"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        },
         "host": {
           "properties": {
             "name": {

x-pack/test/functional/es_archives/security_solution/ecs_non_compliant/mappings.json

@@ -73,6 +73,28 @@
                 "type": "text"
               }
             }
+          },
+          "random": {
+            "properties": {
+              "entry_leader": {
+                "properties": {
+                  "name": {
+                    "type": "keyword",
+                    "ignore_above": 1024,
+                    "fields": {
+                      "text": {
+                        "type": "match_only_text"
+                      },
+                      "caseless": {
+                        "type": "keyword",
+                        "ignore_above": 1024,
+                        "normalizer": "lowercase"
+                      }
+                    }
+                  }
+                }
+              }
+            }
           }
         }
       },