[Security_Solution][Endpoint] Resolver leverage ancestry array for queries

x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts

@@ -10,6 +10,7 @@ import {
   TreeNode,
   RelatedEventCategory,
   ECSCategory,
+  ANCESTRY_LIMIT,
 } from './generate_data';
 
 interface Node {
@@ -113,6 +114,7 @@ describe('data generator', () => {
         relatedEvents: 0,
         relatedAlerts: 0,
       });
+      tree.ancestry.delete(tree.origin.id);
     });
 
     it('creates an alert for the origin node but no other nodes', () => {
@@ -159,6 +161,30 @@ describe('data generator', () => {
       return (inRelated || inRelatedAlerts || inLifecycle) && event.process.entity_id === node.id;
     };
 
+    const verifyAncestry = (event: Event, genTree: Tree) => {
+      if (event.process.Ext.ancestry.length > 0) {
+        expect(event.process.parent?.entity_id).toBe(event.process.Ext.ancestry[0]);
+      }
+      for (let i = 0; i < event.process.Ext.ancestry.length; i++) {
+        const ancestor = event.process.Ext.ancestry[i];
+        const parent = genTree.children.get(ancestor) || genTree.ancestry.get(ancestor);
+        expect(ancestor).toBe(parent?.lifecycle[0].process.entity_id);
+
+        // the next ancestor should be the grandparent
+        if (i + 1 < event.process.Ext.ancestry.length) {
+          const grandparent = event.process.Ext.ancestry[i + 1];
+          expect(grandparent).toBe(parent?.lifecycle[0].process.parent?.entity_id);
+        }
+      }
+    };
+
+    it('has ancestry array defined', () => {
+      expect(tree.origin.lifecycle[0].process.Ext.ancestry.length).toBe(ANCESTRY_LIMIT);
+      for (const event of tree.allEvents) {
+        verifyAncestry(event, tree);
+      }
+    });
+
     it('has the right related events for each node', () => {
       const checkRelatedEvents = (node: TreeNode) => {
         expect(node.relatedEvents.length).toEqual(4);
@@ -185,8 +211,6 @@ describe('data generator', () => {
       for (const node of tree.children.values()) {
         checkRelatedEvents(node);
       }
-
-      checkRelatedEvents(tree.origin);
     });
 
     it('has the right number of related alerts for each node', () => {
@@ -202,7 +226,8 @@ describe('data generator', () => {
     });
 
     it('has the right number of ancestors', () => {
-      expect(tree.ancestry.size).toEqual(ancestors);
+      // +1 for the origin node
+      expect(tree.ancestry.size).toEqual(ancestors + 1);
     });
 
     it('has the right number of total children', () => {
@@ -239,10 +264,7 @@ describe('data generator', () => {
         const children = tree.children.get(event.process.entity_id);
         if (children) {
           expect(eventInNode(event, children)).toBeTruthy();
-          return;
         }
-
-        expect(eventInNode(event, tree.origin)).toBeTruthy();
       });
     });
 
@@ -260,8 +282,6 @@ describe('data generator', () => {
         total += nodeEventCount(node);
       }
 
-      total += nodeEventCount(tree.origin);
-
       expect(tree.allEvents.length).toEqual(total);
     });
   });

x-pack/plugins/security_solution/common/endpoint/generate_data.ts

@@ -18,6 +18,7 @@ import {
 import { factory as policyFactory } from './models/policy_config';
 
 export type Event = AlertEvent | EndpointEvent;
+export const ANCESTRY_LIMIT: number = 2;
 
 interface EventOptions {
   timestamp?: number;
@@ -26,6 +27,7 @@ interface EventOptions {
   eventType?: string;
   eventCategory?: string | string[];
   processName?: string;
+  ancestry?: string[];
 }
 
 const Windows: HostOS[] = [
@@ -238,6 +240,7 @@ export interface Tree {
    * Map of entity_id to node
    */
   ancestry: Map<string, TreeNode>;
+  // TODO add the origin to the ancestry array to make test verification easier
   origin: TreeNode;
   /**
    * All events from children, ancestry, origin, and the alert in a single array
@@ -341,7 +344,9 @@ export class EndpointDocGenerator {
   public generateAlert(
     ts = new Date().getTime(),
     entityID = this.randomString(10),
-    parentEntityID?: string
+    parentEntityID?: string,
+    ancestryArray: string[] = [],
+    ancestryLimit: number = 2
   ): AlertEvent {
     return {
       ...this.commonInfo,
@@ -416,6 +421,9 @@ export class EndpointDocGenerator {
           sha1: 'fake sha1',
           sha256: 'fake sha256',
         },
+        // simulate a finite ancestry array size, the endpoint limits the ancestry array to 20 entries we'll use
+        // 2 so that the backend can handle that case
+        Ext: { ancestry: ancestryArray.slice(0, ANCESTRY_LIMIT) },
       },
       dll: [
         {
@@ -469,6 +477,9 @@ export class EndpointDocGenerator {
         entity_id: options.entityID ? options.entityID : this.randomString(10),
         parent: options.parentEntityID ? { entity_id: options.parentEntityID } : undefined,
         name: options.processName ? options.processName : randomProcessName(),
+        // simulate a finite ancestry array size, the endpoint limits the ancestry array to 20 entries we'll use
+        // 2 so that the backend can handle that case
+        Ext: { ancestry: options.ancestry?.slice(0, ANCESTRY_LIMIT) || [] },
       },
     };
   }
@@ -522,9 +533,6 @@ export class EndpointDocGenerator {
       throw Error(`could not find origin while building tree: ${alert.process.entity_id}`);
     }
 
-    // remove the origin node from the ancestry array
-    ancestryNodes.delete(alert.process.entity_id);
-
     const children = Array.from(
       this.descendantsTreeGenerator(
         alert,
@@ -676,7 +684,7 @@ export class EndpointDocGenerator {
       }
     };
 
-    // generate related alerts for rootW
+    // generate related alerts for root
     const processDuration: number = 6 * 3600;
     if (this.randomN(100) < pctWithRelated) {
       addRelatedEvents(ancestor, processDuration, events);
@@ -701,6 +709,8 @@ export class EndpointDocGenerator {
       ancestor = this.generateEvent({
         timestamp,
         parentEntityID: ancestor.process.entity_id,
+        // add the parent to the ancestry array
+        ancestry: [ancestor.process.entity_id, ...ancestor.process.Ext.ancestry],
       });
       events.push(ancestor);
       timestamp = timestamp + 1000;
@@ -714,6 +724,7 @@ export class EndpointDocGenerator {
             parentEntityID: ancestor.process.parent?.entity_id,
             eventCategory: 'process',
             eventType: 'end',
+            ancestry: ancestor.process.Ext.ancestry,
           })
         );
       }
@@ -732,7 +743,12 @@ export class EndpointDocGenerator {
       }
     }
     events.push(
-      this.generateAlert(timestamp, ancestor.process.entity_id, ancestor.process.parent?.entity_id)
+      this.generateAlert(
+        timestamp,
+        ancestor.process.entity_id,
+        ancestor.process.parent?.entity_id,
+        ancestor.process.Ext.ancestry
+      )
     );
     return events;
   }
@@ -787,6 +803,10 @@ export class EndpointDocGenerator {
       const child = this.generateEvent({
         timestamp,
         parentEntityID: currentState.event.process.entity_id,
+        ancestry: [
+          currentState.event.process.entity_id,
+          ...currentState.event.process.Ext.ancestry,
+        ],
       });
 
       maxChildren = this.randomN(maxChildrenPerNode + 1);
@@ -808,6 +828,7 @@ export class EndpointDocGenerator {
           parentEntityID: child.process.parent?.entity_id,
           eventCategory: 'process',
           eventType: 'end',
+          ancestry: child.process.Ext.ancestry,
         });
       }
       if (this.randomN(100) < percentNodesWithRelated) {
@@ -852,6 +873,7 @@ export class EndpointDocGenerator {
           parentEntityID: node.process.parent?.entity_id,
           eventCategory: eventInfo.category,
           eventType: eventInfo.creationType,
+          ancestry: node.process.Ext.ancestry,
         });
       }
     }
@@ -870,7 +892,12 @@ export class EndpointDocGenerator {
   ) {
     for (let i = 0; i < relatedAlerts; i++) {
       const ts = node['@timestamp'] + this.randomN(alertCreationTime) * 1000;
-      yield this.generateAlert(ts, node.process.entity_id, node.process.parent?.entity_id);
+      yield this.generateAlert(
+        ts,
+        node.process.entity_id,
+        node.process.parent?.entity_id,
+        node.process.Ext.ancestry
+      );
     }
   }
 

x-pack/plugins/security_solution/common/endpoint/models/event.ts

@@ -52,3 +52,10 @@ export function parentEntityId(event: ResolverEvent): string | undefined {
   }
   return event.process.parent?.entity_id;
 }
+
+export function ancestryArray(event: ResolverEvent): string[] {
+  if (isLegacyEvent(event)) {
+    return [];
+  }
+  return event.process.Ext.ancestry;
+}

x-pack/plugins/security_solution/common/endpoint/types.ts

@@ -313,6 +313,9 @@ export interface AlertEvent {
     thread?: ThreadFields[];
     uptime: number;
     user: string;
+    Ext: {
+      ancestry: string[];
+    };
   };
   file: {
     owner: string;
@@ -445,6 +448,9 @@ export interface EndpointEvent {
       entity_id: string;
       name?: string;
     };
+    Ext: {
+      ancestry: string[];
+    };
   };
 }