Add Internal Alert/Signal ID to Endpoint Alert telemetry (#126216) (#…

…126739)

* Attach the internal signal_id to the endpoint alert to join with insights

* Ensure we forward signal_id field properly

* Don't think we need to explicitly define the field in the top-level since it satisfies the key:string

* Updated unit test to check for signal id enrichment

* Addressed some comments about alert_id enrichment

* Refactored send_telemetry_events to be more performant and idiomatic

* Added test cases with a non-matching enrichment or non-existing enrichment

* Broke some tests that don't assume QueueTelemetryEvents are endpoint.alerts

* my types were still off

* Addressed comments to use more idiomatic 'toString' function

* Fixed 'Cannot access signalIdMap before initialization name' in reduce by instatiating map prior to reduce

Co-authored-by: Kibana Machine <[email protected]>
(cherry picked from commit 2f083bf)

Co-authored-by: Chris Donaher <[email protected]>

x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts

@@ -176,7 +176,13 @@ export const searchAfterAndBulkCreate = async ({
             buildRuleMessage(`enrichedEvents.hits.hits: ${enrichedEvents.hits.hits.length}`)
           );
 
-          sendAlertTelemetryEvents(logger, eventsTelemetry, enrichedEvents, buildRuleMessage);
+          sendAlertTelemetryEvents(
+            logger,
+            eventsTelemetry,
+            enrichedEvents,
+            createdItems,
+            buildRuleMessage
+          );
         }
 
         if (!hasSortId) {

x-pack/plugins/security_solution/server/lib/detection_engine/signals/send_telemetry_events.test.ts

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-import { selectEvents } from './send_telemetry_events';
+import { selectEvents, enrichEndpointAlertsSignalID } from './send_telemetry_events';
 
 describe('sendAlertTelemetry', () => {
   it('selectEvents', () => {
@@ -33,6 +33,9 @@ describe('sendAlertTelemetry', () => {
               data_stream: {
                 dataset: 'endpoint.events',
               },
+              event: {
+                id: 'foo',
+              },
             },
           },
           {
@@ -47,6 +50,9 @@ describe('sendAlertTelemetry', () => {
                 dataset: 'endpoint.alerts',
                 other: 'x',
               },
+              event: {
+                id: 'bar',
+              },
             },
           },
           {
@@ -58,13 +64,52 @@ describe('sendAlertTelemetry', () => {
               '@timestamp': 'x',
               key3: 'hello',
               data_stream: {},
+              event: {
+                id: 'baz',
+              },
+            },
+          },
+          {
+            _index: 'y',
+            _type: 'y',
+            _id: 'y',
+            _score: 0,
+            _source: {
+              '@timestamp': 'y',
+              key3: 'hello',
+              data_stream: {
+                dataset: 'endpoint.alerts',
+                other: 'y',
+              },
+              event: {
+                id: 'not-in-map',
+              },
+            },
+          },
+          {
+            _index: 'z',
+            _type: 'z',
+            _id: 'z',
+            _score: 0,
+            _source: {
+              '@timestamp': 'z',
+              key3: 'no-event-id',
+              data_stream: {
+                dataset: 'endpoint.alerts',
+                other: 'z',
+              },
             },
           },
         ],
       },
     };
-
-    const sources = selectEvents(filteredEvents);
+    const joinMap = new Map<string, string>([
+      ['foo', '1234'],
+      ['bar', 'abcd'],
+      ['baz', '4567'],
+    ]);
+    const subsetEvents = selectEvents(filteredEvents);
+    const sources = enrichEndpointAlertsSignalID(subsetEvents, joinMap);
     expect(sources).toStrictEqual([
       {
         '@timestamp': 'x',
@@ -73,6 +118,31 @@ describe('sendAlertTelemetry', () => {
           dataset: 'endpoint.alerts',
           other: 'x',
         },
+        event: {
+          id: 'bar',
+        },
+        signal_id: 'abcd',
+      },
+      {
+        '@timestamp': 'y',
+        key3: 'hello',
+        data_stream: {
+          dataset: 'endpoint.alerts',
+          other: 'y',
+        },
+        event: {
+          id: 'not-in-map',
+        },
+        signal_id: undefined,
+      },
+      {
+        '@timestamp': 'z',
+        key3: 'no-event-id',
+        data_stream: {
+          dataset: 'endpoint.alerts',
+          other: 'z',
+        },
+        signal_id: undefined,
       },
     ]);
   });

x-pack/plugins/security_solution/server/lib/detection_engine/signals/send_telemetry_events.ts

@@ -11,14 +11,17 @@ import { BuildRuleMessage } from './rule_messages';
 import { SignalSearchResponse, SignalSource } from './types';
 import { Logger } from '../../../../../../../src/core/server';
 
-export interface SearchResultWithSource {
+interface SearchResultSource {
   _source: SignalSource;
 }
 
+type CreatedSignalId = string;
+type AlertId = string;
+
 export function selectEvents(filteredEvents: SignalSearchResponse): TelemetryEvent[] {
   // @ts-expect-error @elastic/elasticsearch _source is optional
   const sources: TelemetryEvent[] = filteredEvents.hits.hits.map(function (
-    obj: SearchResultWithSource
+    obj: SearchResultSource
   ): TelemetryEvent {
     return obj._source;
   });
@@ -27,20 +30,49 @@ export function selectEvents(filteredEvents: SignalSearchResponse): TelemetryEve
   return sources.filter((obj: TelemetryEvent) => obj.data_stream?.dataset === 'endpoint.alerts');
 }
 
+export function enrichEndpointAlertsSignalID(
+  events: TelemetryEvent[],
+  signalIdMap: Map<string, string>
+): TelemetryEvent[] {
+  return events.map(function (obj: TelemetryEvent): TelemetryEvent {
+    obj.signal_id = undefined;
+    if (obj?.event?.id !== undefined) {
+      obj.signal_id = signalIdMap.get(obj.event.id);
+    }
+    return obj;
+  });
+}
+
 export function sendAlertTelemetryEvents(
   logger: Logger,
   eventsTelemetry: ITelemetryEventsSender | undefined,
   filteredEvents: SignalSearchResponse,
+  createdEvents: SignalSource[],
   buildRuleMessage: BuildRuleMessage
 ) {
   if (eventsTelemetry === undefined) {
     return;
   }
 
-  const sources = selectEvents(filteredEvents);
+  let selectedEvents = selectEvents(filteredEvents);
+  if (selectedEvents.length > 0) {
+    // Create map of ancenstor_id -> alert_id
+    let signalIdMap = new Map<CreatedSignalId, AlertId>();
+    /* eslint-disable no-param-reassign */
+    signalIdMap = createdEvents.reduce((signalMap, obj) => {
+      const ancestorId = obj['kibana.alert.original_event.id']?.toString();
+      const alertId = obj._id?.toString();
+      if (ancestorId !== null && ancestorId !== undefined && alertId !== undefined) {
+        signalMap = signalIdMap.set(ancestorId, alertId);
+      }
+
+      return signalMap;
+    }, new Map<CreatedSignalId, AlertId>());
 
+    selectedEvents = enrichEndpointAlertsSignalID(selectedEvents, signalIdMap);
+  }
   try {
-    eventsTelemetry.queueTelemetryEvents(sources);
+    eventsTelemetry.queueTelemetryEvents(selectedEvents);
   } catch (exc) {
     logger.error(buildRuleMessage(`[-] queing telemetry events failed ${exc}`));
   }

x-pack/plugins/security_solution/server/lib/telemetry/filters.ts

@@ -108,6 +108,7 @@ const allowlistBaseEventFields: AllowlistFields = {
 export const allowlistEventFields: AllowlistFields = {
   _id: true,
   '@timestamp': true,
+  signal_id: true,
   agent: true,
   Endpoint: true,
   /* eslint-disable @typescript-eslint/naming-convention */

x-pack/plugins/security_solution/server/lib/telemetry/sender.test.ts

@@ -35,6 +35,7 @@ describe('TelemetryEventsSender', () => {
         {
           event: {
             kind: 'alert',
+            id: 'test',
           },
           dns: {
             question: {
@@ -108,6 +109,7 @@ describe('TelemetryEventsSender', () => {
         {
           event: {
             kind: 'alert',
+            id: 'test',
           },
           dns: {
             question: {

x-pack/plugins/security_solution/server/lib/telemetry/types.ts

@@ -58,6 +58,10 @@ export interface TelemetryEvent {
     };
   };
   license?: ESLicense;
+  event?: {
+    id?: string;
+    kind?: string;
+  };
 }
 
 // EP Policy Response