[Endpoint] EDVT-72 EDVT-25 Initial Resolver Backend Refactored (#57593)

* Stubbed route, static tree
Changing commit author to me

* Starting the queries

* Built out children api

* using agent id instead of labels and implementing the get node route

* Adding pagination, need to write tests

* Removing track_total_hits because it defaults to 10k

* Allowing null for origin information

* Building out tests

* Adding more response tests

* Adding test for children route

* Forgot to save

* Reverting first commit and keeping my changes to resolver route

* Working search handler tests

* Adding api test

* Trying to figure out the query issue

* Working api tests

* A little refactoring of common types

* Fixing tests

* Some clean up and fixing bad merge

* Working api

* Changing phse0 names

* Refactoring duplicate code in route

* Adding test for count query and fixing api test

* Renaming phase 1 to elastic endpoint

* Removing test files without tests

* Restructuring things

* Building events for process handler

* Working unit tests

* Working integration tests

* Pagination test is failing

* More refactoring

* Add alternative fields to support other datasources

* Working refactored routes

* Strip out 'synthetic id' stuff

* allow ppid to be undefined

* Some clean up and fixing typescript lint errors

* Removing changes to alerts

* Remove the additional query with the _id cursor

* Use Buffer.from instead of new Buffer

* Fixing import

* Fix decoding

* Fix Promise return type

* Fixing linter used before assigned problem

* Removing unused import

* gzipping the test file

* Addressing feedback, more clean next cursor

* Fixing cast for search_after

* Fixing test failure and adding comments

* Fixing timestamp string type failure

Co-authored-by: Elastic Machine <[email protected]>
Co-authored-by: Andrew Stucki <[email protected]>

x-pack/plugins/endpoint/common/types.ts

@@ -25,6 +25,11 @@ export type ImmutableObject<T> = { readonly [K in keyof T]: Immutable<T[K]> };
 export class EndpointAppConstants {
   static ALERT_INDEX_NAME = 'my-index';
   static ENDPOINT_INDEX_NAME = 'endpoint-agent*';
+  static EVENT_INDEX_NAME = 'endpoint-events-*';
+  /**
+   * Legacy events are stored in indices with endgame-* prefix
+   */
+  static LEGACY_EVENT_INDEX_NAME = 'endgame-*';
 }
 
 export interface AlertResultList {
@@ -117,6 +122,43 @@ export interface EndpointMetadata {
   };
 }
 
+export interface LegacyEndpointEvent {
+  '@timestamp': Date;
+  endgame: {
+    event_type_full: string;
+    event_subtype_full: string;
+    unique_pid: number;
+    unique_ppid: number;
+    serial_event_id: number;
+  };
+  agent: {
+    id: string;
+    type: string;
+  };
+}
+
+export interface EndpointEvent {
+  '@timestamp': Date;
+  event: {
+    category: string;
+    type: string;
+    id: string;
+  };
+  endpoint: {
+    process: {
+      entity_id: string;
+      parent: {
+        entity_id: string;
+      };
+    };
+  };
+  agent: {
+    type: string;
+  };
+}
+
+export type ResolverEvent = EndpointEvent | LegacyEndpointEvent;
+
 /**
  * The PageId type is used for the payload when firing userNavigatedToPage actions
  */

x-pack/plugins/endpoint/server/plugin.ts

@@ -5,12 +5,14 @@
  */
 import { Plugin, CoreSetup, PluginInitializerContext, Logger } from 'kibana/server';
 import { first } from 'rxjs/operators';
-import { addRoutes } from './routes';
 import { PluginSetupContract as FeaturesPluginSetupContract } from '../../features/server';
 import { createConfig$, EndpointConfigType } from './config';
-import { registerEndpointRoutes } from './routes/endpoints';
 import { EndpointAppContext } from './types';
+
+import { addRoutes } from './routes';
+import { registerEndpointRoutes } from './routes/endpoints';
 import { registerAlertRoutes } from './routes/alerts';
+import { registerResolverRoutes } from './routes/resolver';
 
 export type EndpointPluginStart = void;
 export type EndpointPluginSetup = void;
@@ -69,6 +71,7 @@ export class EndpointPlugin
     const router = core.http.createRouter();
     addRoutes(router);
     registerEndpointRoutes(router, endpointContext);
+    registerResolverRoutes(router, endpointContext);
     registerAlertRoutes(router, endpointContext);
   }
 

x-pack/plugins/endpoint/server/routes/resolver.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { IRouter } from 'kibana/server';
+import { EndpointAppContext } from '../types';
+import { handleRelatedEvents, validateRelatedEvents } from './resolver/related_events';
+import { handleChildren, validateChildren } from './resolver/children';
+import { handleLifecycle, validateLifecycle } from './resolver/lifecycle';
+
+export function registerResolverRoutes(router: IRouter, endpointAppContext: EndpointAppContext) {
+  const log = endpointAppContext.logFactory.get('resolver');
+
+  router.get(
+    {
+      path: '/api/endpoint/resolver/{id}/related',
+      validate: validateRelatedEvents,
+      options: { authRequired: true },
+    },
+    handleRelatedEvents(log)
+  );
+
+  router.get(
+    {
+      path: '/api/endpoint/resolver/{id}/children',
+      validate: validateChildren,
+      options: { authRequired: true },
+    },
+    handleChildren(log)
+  );
+
+  router.get(
+    {
+      path: '/api/endpoint/resolver/{id}',
+      validate: validateLifecycle,
+      options: { authRequired: true },
+    },
+    handleLifecycle(log)
+  );
+}

x-pack/plugins/endpoint/server/routes/resolver/children.ts

@@ -0,0 +1,90 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import _ from 'lodash';
+import { schema } from '@kbn/config-schema';
+import { RequestHandler, Logger } from 'kibana/server';
+import { extractEntityID } from './utils/normalize';
+import { getPaginationParams } from './utils/pagination';
+import { LifecycleQuery } from './queries/lifecycle';
+import { ChildrenQuery } from './queries/children';
+
+interface ChildrenQueryParams {
+  after?: string;
+  limit: number;
+  /**
+   * legacyEndpointID is optional because there are two different types of identifiers:
+   *
+   * Legacy
+   * A legacy Entity ID is made up of the agent.id and unique_pid fields. The client will need to identify if
+   * it's looking at a legacy event and use those fields when making requests to the backend. The
+   * request would be /resolver/{id}?legacyEndpointID=<some uuid>and the {id} would be the unique_pid.
+   *
+   * Elastic Endpoint
+   * When interacting with the new form of data the client doesn't need the legacyEndpointID because it's already a
+   * part of the entityID in the new type of event. So for the same request the client would just hit resolver/{id}
+   * and the {id} would be entityID stored in the event's process.entity_id field.
+   */
+  legacyEndpointID?: string;
+}
+
+interface ChildrenPathParams {
+  id: string;
+}
+
+export const validateChildren = {
+  params: schema.object({ id: schema.string() }),
+  query: schema.object({
+    after: schema.maybe(schema.string()),
+    limit: schema.number({ defaultValue: 10, min: 1, max: 100 }),
+    legacyEndpointID: schema.maybe(schema.string()),
+  }),
+};
+
+export function handleChildren(
+  log: Logger
+): RequestHandler<ChildrenPathParams, ChildrenQueryParams> {
+  return async (context, req, res) => {
+    const {
+      params: { id },
+      query: { limit, after, legacyEndpointID },
+    } = req;
+    try {
+      const pagination = getPaginationParams(limit, after);
+
+      const client = context.core.elasticsearch.dataClient;
+      const childrenQuery = new ChildrenQuery(legacyEndpointID, pagination);
+      const lifecycleQuery = new LifecycleQuery(legacyEndpointID);
+
+      // Retrieve the related child process events for a given process
+      const { total, results: events, nextCursor } = await childrenQuery.search(client, id);
+      const childIDs = events.map(extractEntityID);
+
+      // Retrieve the lifecycle events for the child processes (e.g. started, terminated etc)
+      // this needs to fire after the above since we don't yet have the entity ids until we
+      // run the first query
+      const { results: lifecycleEvents } = await lifecycleQuery.search(client, ...childIDs);
+
+      // group all of the lifecycle events by the child process id
+      const lifecycleGroups = Object.values(_.groupBy(lifecycleEvents, extractEntityID));
+      const children = lifecycleGroups.map(group => ({ lifecycle: group }));
+
+      return res.ok({
+        body: {
+          children,
+          pagination: {
+            total,
+            next: nextCursor,
+            limit,
+          },
+        },
+      });
+    } catch (err) {
+      log.warn(err);
+      return res.internalError({ body: err });
+    }
+  };
+}

x-pack/plugins/endpoint/server/routes/resolver/lifecycle.ts

@@ -0,0 +1,94 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import _ from 'lodash';
+import { schema } from '@kbn/config-schema';
+import { RequestHandler, Logger } from 'kibana/server';
+import { extractParentEntityID } from './utils/normalize';
+import { LifecycleQuery } from './queries/lifecycle';
+import { ResolverEvent } from '../../../common/types';
+
+interface LifecycleQueryParams {
+  ancestors: number;
+  /**
+   * legacyEndpointID is optional because there are two different types of identifiers:
+   *
+   * Legacy
+   * A legacy Entity ID is made up of the agent.id and unique_pid fields. The client will need to identify if
+   * it's looking at a legacy event and use those fields when making requests to the backend. The
+   * request would be /resolver/{id}?legacyEndpointID=<some uuid>and the {id} would be the unique_pid.
+   *
+   * Elastic Endpoint
+   * When interacting with the new form of data the client doesn't need the legacyEndpointID because it's already a
+   * part of the entityID in the new type of event. So for the same request the client would just hit resolver/{id}
+   * and the {id} would be entityID stored in the event's process.entity_id field.
+   */
+  legacyEndpointID?: string;
+}
+
+interface LifecyclePathParams {
+  id: string;
+}
+
+export const validateLifecycle = {
+  params: schema.object({ id: schema.string() }),
+  query: schema.object({
+    ancestors: schema.number({ defaultValue: 0, min: 0, max: 10 }),
+    legacyEndpointID: schema.maybe(schema.string()),
+  }),
+};
+
+function getParentEntityID(results: ResolverEvent[]) {
+  return results.length === 0 ? undefined : extractParentEntityID(results[0]);
+}
+
+export function handleLifecycle(
+  log: Logger
+): RequestHandler<LifecyclePathParams, LifecycleQueryParams> {
+  return async (context, req, res) => {
+    const {
+      params: { id },
+      query: { ancestors, legacyEndpointID },
+    } = req;
+    try {
+      const ancestorLifecycles = [];
+      const client = context.core.elasticsearch.dataClient;
+
+      const lifecycleQuery = new LifecycleQuery(legacyEndpointID);
+      const { results: processLifecycle } = await lifecycleQuery.search(client, id);
+      let nextParentID = getParentEntityID(processLifecycle);
+
+      if (nextParentID) {
+        for (let i = 0; i < ancestors; i++) {
+          const { results: lifecycle } = await lifecycleQuery.search(client, nextParentID);
+          nextParentID = getParentEntityID(lifecycle);
+
+          if (!nextParentID) {
+            break;
+          }
+
+          ancestorLifecycles.push({
+            lifecycle,
+          });
+        }
+      }
+
+      return res.ok({
+        body: {
+          lifecycle: processLifecycle,
+          ancestors: ancestorLifecycles,
+          pagination: {
+            next: nextParentID || null,
+            ancestors,
+          },
+        },
+      });
+    } catch (err) {
+      log.warn(err);
+      return res.internalError({ body: err });
+    }
+  };
+}

x-pack/plugins/endpoint/server/routes/resolver/queries/base.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { IScopedClusterClient } from 'kibana/server';
+import { EndpointAppConstants } from '../../../../common/types';
+import { paginate, paginatedResults, PaginationParams } from '../utils/pagination';
+import { JsonObject } from '../../../../../../../src/plugins/kibana_utils/public';
+
+export abstract class ResolverQuery {
+  constructor(
+    private readonly endpointID?: string,
+    private readonly pagination?: PaginationParams
+  ) {}
+
+  protected paginateBy(field: string, query: JsonObject) {
+    if (!this.pagination) {
+      return query;
+    }
+    return paginate(this.pagination, field, query);
+  }
+
+  build(...ids: string[]) {
+    if (this.endpointID) {
+      return this.legacyQuery(this.endpointID, ids, EndpointAppConstants.LEGACY_EVENT_INDEX_NAME);
+    }
+    return this.query(ids, EndpointAppConstants.EVENT_INDEX_NAME);
+  }
+
+  async search(client: IScopedClusterClient, ...ids: string[]) {
+    return paginatedResults(await client.callAsCurrentUser('search', this.build(...ids)));
+  }
+
+  protected abstract legacyQuery(
+    endpointID: string,
+    uniquePIDs: string[],
+    index: string
+  ): JsonObject;
+  protected abstract query(entityIDs: string[], index: string): JsonObject;
+}