Risk engine initialisation, update from legacy risk engine workflow a…

…nd status change (elastic#162400) ## Risk engine initialisation, update from legacy risk engine workflow and status change https://github.com/elastic/kibana/assets/7609147/dfb75d4a-f447-4346-9760-d0e9685cce39 Green areas it is what was implemented <img width="1449" alt="Screenshot 2023-08-01 at 15 07 01" src="https://github.com/elastic/kibana/assets/7609147/4d87887f-1163-45eb-a4e9-a77a685f6565"> This pr has: - Upgrade workflow. If the user has a risk host or user transforms, we will show the panel with a call to action for the upgrade. - Introduce new Saved object to save the configuration of risk engine - API which is described bellow It required experiment enabled - **riskScoringRoutesEnabled** ## New API ### /engine/status #### GET Get the status of the Risk Engine ##### Description: Returns the status of both the legacy transform-based risk engine, as well as the new risk engine ##### Responses ```json { "legacy_risk_engine_status": "NOT_INSTALLED" , "ENABLED" , "risk_engine_status": "NOT_INSTALLED" , "ENABLED" , "DISABLED" } ``` ### /engine/init #### POST Initialize the Risk Engine ##### Description: Initializes the Risk Engine by creating the necessary indices and mappings, removing old transforms, creating saved object configuration ##### Responses ```json { "result": { "risk_engine_enabled": true, "risk_engine_resources_installed": true, "risk_engine_configuration_created": true, "legacy_risk_engine_disabled": true, "errors": [ "string" ] } } ``` ### /engine/enable #### POST Enable the Risk Engine ##### Description: Change saved object configuration and in the future here we will start task ### /engine/disable #### POST Disable the Risk Engine Change saved object configuration and in the future here we will stop task --------- Co-authored-by: kibanamachine <[email protected]> Co-authored-by: Ryland Herrick <[email protected]>
pmuellr · Aug 4, 2023 · 2bd52fc · 2bd52fc
1 parent 0144696
commit 2bd52fc
Show file tree

Hide file tree

Showing 48 changed files with 2,498 additions and 360 deletions.
diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json
@@ -42,6 +42,25 @@
       }
     }
   },
+  "url": {
+    "dynamic": false,
+    "properties": {
+      "slug": {
+        "type": "text",
+        "fields": {
+          "keyword": {
+            "type": "keyword"
+          }
+        }
+      },
+      "accessDate": {
+        "type": "date"
+      },
+      "createDate": {
+        "type": "date"
+      }
+    }
+  },
   "usage-counters": {
     "dynamic": false,
     "properties": {
@@ -131,25 +150,6 @@
       }
     }
   },
-  "url": {
-    "dynamic": false,
-    "properties": {
-      "slug": {
-        "type": "text",
-        "fields": {
-          "keyword": {
-            "type": "keyword"
-          }
-        }
-      },
-      "accessDate": {
-        "type": "date"
-      },
-      "createDate": {
-        "type": "date"
-      }
-    }
-  },
   "index-pattern": {
     "dynamic": false,
     "properties": {
@@ -1407,6 +1407,14 @@
     "dynamic": false,
     "properties": {}
   },
+  "infrastructure-monitoring-log-view": {
+    "dynamic": false,
+    "properties": {
+      "name": {
+        "type": "text"
+      }
+    }
+  },
   "canvas-element": {
     "dynamic": false,
     "properties": {
@@ -2262,14 +2270,6 @@
       }
     }
   },
-  "infrastructure-monitoring-log-view": {
-    "dynamic": false,
-    "properties": {
-      "name": {
-        "type": "text"
-      }
-    }
-  },
   "ml-job": {
     "properties": {
       "job_id": {
@@ -2938,6 +2938,14 @@
       }
     }
   },
+  "risk-engine-configuration": {
+    "dynamic": false,
+    "properties": {
+      "enabled": {
+        "type": "boolean"
+      }
+    }
+  },
   "infrastructure-ui-source": {
     "dynamic": false,
     "properties": {}

diff --git a/packages/kbn-test/src/kbn_client/kbn_client_saved_objects.ts b/packages/kbn-test/src/kbn_client/kbn_client_saved_objects.ts
@@ -8,7 +8,10 @@
 
 import { chunk } from 'lodash';
 import type { ToolingLog } from '@kbn/tooling-log';
-import type { SavedObjectsBulkDeleteResponse } from '@kbn/core-saved-objects-api-server';
+import type {
+  SavedObjectsBulkDeleteResponse,
+  SavedObjectsFindResponse,
+} from '@kbn/core-saved-objects-api-server';
 
 import { KbnClientRequester, uriencode } from './kbn_client_requester';
 
@@ -30,6 +33,11 @@ interface SavedObjectResponse<Attributes extends Record<string, any>> {
   version?: string;
 }
 
+interface FindOptions {
+  type: string;
+  space?: string;
+}
+
 interface GetOptions {
   type: string;
   id: string;
@@ -152,6 +160,22 @@ export class KbnClientSavedObjects {
     return data;
   }
 
+  /**
+   * Find saved objects
+   */
+  public async find<Attributes extends Record<string, any>>(options: FindOptions) {
+    this.log.debug('Find saved objects: %j', options);
+
+    const { data } = await this.requester.request<SavedObjectsFindResponse<Attributes>>({
+      description: 'find saved objects',
+      path: options.space
+        ? uriencode`/s/${options.space}/internal/ftr/kbn_client_so/_find?type=${options.type}`
+        : uriencode`/internal/ftr/kbn_client_so/_find?type=${options.type}`,
+      method: 'GET',
+    });
+    return data;
+  }
+
   /**
    * Create a saved object
    */

diff --git a/...e/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts b/...e/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts
@@ -126,6 +126,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "osquery-pack-asset": "b14101d3172c4b60eb5404696881ce5275c84152",
         "osquery-saved-query": "44f1161e165defe3f9b6ad643c68c542a765fcdb",
         "query": "8db5d48c62d75681d80d82a42b5642f60d068202",
+        "risk-engine-configuration": "1b8b175e29ea5311408125c92c6247f502b2d79d",
         "rules-settings": "892a2918ebaeba809a612b8d97cec0b07c800b5f",
         "sample-data-telemetry": "37441b12f5b0159c2d6d5138a494c9f440e950b5",
         "search": "8d5184dd5b986d57250b6ffd9ae48a1925e4c7a3",

diff --git a/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group3/type_registrations.test.ts
@@ -104,6 +104,7 @@ const previouslyRegisteredTypes = [
   'search-telemetry',
   'security-rule',
   'security-solution-signals-migration',
+  'risk-engine-configuration',
   'server',
   'siem-detection-engine-rule-actions',
   'siem-detection-engine-rule-execution-info',

diff --git a/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group5/dot_kibana_split.test.ts
@@ -246,6 +246,7 @@ describe('split .kibana index into multiple system indices', () => {
             "osquery-pack-asset",
             "osquery-saved-query",
             "query",
+            "risk-engine-configuration",
             "rules-settings",
             "sample-data-telemetry",
             "search-session",

diff --git a/x-pack/plugins/security_solution/common/constants.ts b/x-pack/plugins/security_solution/common/constants.ts
@@ -253,6 +253,12 @@ export const RISK_SCORE_CREATE_STORED_SCRIPT = `${INTERNAL_RISK_SCORE_URL}/store
 export const RISK_SCORE_DELETE_STORED_SCRIPT = `${INTERNAL_RISK_SCORE_URL}/stored_scripts/delete`;
 export const RISK_SCORE_PREVIEW_URL = `${INTERNAL_RISK_SCORE_URL}/preview`;
 
+export const RISK_ENGINE_URL = `${INTERNAL_RISK_SCORE_URL}/engine`;
+export const RISK_ENGINE_STATUS_URL = `${RISK_ENGINE_URL}/status`;
+export const RISK_ENGINE_INIT_URL = `${RISK_ENGINE_URL}/init`;
+export const RISK_ENGINE_ENABLE_URL = `${RISK_ENGINE_URL}/enable`;
+export const RISK_ENGINE_DISABLE_URL = `${RISK_ENGINE_URL}/disable`;
+
 /**
  * Public Risk Score routes
  */

diff --git a/x-pack/plugins/security_solution/common/risk_engine/types.ts b/x-pack/plugins/security_solution/common/risk_engine/types.ts
@@ -9,3 +9,17 @@ export enum RiskScoreEntity {
   host = 'host',
   user = 'user',
 }
+
+export enum RiskEngineStatus {
+  NOT_INSTALLED = 'NOT_INSTALLED',
+  DISABLED = 'DISABLED',
+  ENABLED = 'ENABLED',
+}
+
+export interface InitRiskEngineResult {
+  legacyRiskEngineDisabled: boolean;
+  riskEngineResourcesInstalled: boolean;
+  riskEngineConfigurationCreated: boolean;
+  riskEngineEnabled: boolean;
+  errors: string[];
+}
diff --git a/...ins/security_solution/cypress/e2e/entity_analytics/entity_analytics_management_page.cy.ts b/...ins/security_solution/cypress/e2e/entity_analytics/entity_analytics_management_page.cy.ts
@@ -12,83 +12,142 @@ import {
   USER_RISK_PREVIEW_TABLE,
   USER_RISK_PREVIEW_TABLE_ROWS,
   RISK_PREVIEW_ERROR,
-  RISK_PREVIEW_ERROR_BUTTON,
   LOCAL_QUERY_BAR_SELECTOR,
+  RISK_SCORE_ERROR_PANEL,
+  RISK_SCORE_STATUS,
 } from '../../screens/entity_analytics_management';
 
+import { deleteRiskScore, installRiskScoreModule } from '../../tasks/api_calls/risk_scores';
+import { RiskScoreEntity } from '../../tasks/risk_scores/common';
 import { login, visit, visitWithoutDateRange } from '../../tasks/login';
 import { cleanKibana } from '../../tasks/common';
 import { ENTITY_ANALYTICS_MANAGEMENT_URL, ALERTS_URL } from '../../urls/navigation';
 import { getNewRule } from '../../objects/rule';
 import { createRule } from '../../tasks/api_calls/rules';
+import {
+  deleteConfiguration,
+  interceptRiskPreviewError,
+  interceptRiskPreviewSuccess,
+  interceptRiskInitError,
+} from '../../tasks/api_calls/risk_engine';
 import { updateDateRangeInLocalDatePickers } from '../../tasks/date_picker';
 import { fillLocalSearchBar, submitLocalSearch } from '../../tasks/search_bar';
+import {
+  riskEngineStatusChange,
+  updateRiskEngine,
+  updateRiskEngineConfirm,
+  previewErrorButtonClick,
+} from '../../tasks/entity_analytics';
+
+describe(
+  'Entity analytics management page',
+  { env: { ftrConfig: { enableExperimental: ['riskScoringRoutesEnabled'] } } },
+  () => {
+    before(() => {
+      cleanKibana();
+      cy.task('esArchiverLoad', 'all_users');
+    });
 
-describe('Entity analytics management page', () => {
-  before(() => {
-    cleanKibana();
-    cy.task('esArchiverLoad', 'all_users');
-  });
+    beforeEach(() => {
+      login();
+      visitWithoutDateRange(ALERTS_URL);
+      createRule(getNewRule({ query: 'user.name:* or host.name:*', risk_score: 70 }));
+      deleteConfiguration();
+      visit(ENTITY_ANALYTICS_MANAGEMENT_URL);
+    });
 
-  beforeEach(() => {
-    login();
-    visitWithoutDateRange(ALERTS_URL);
-    createRule(getNewRule({ query: 'user.name:* or host.name:*', risk_score: 70 }));
-    visit(ENTITY_ANALYTICS_MANAGEMENT_URL);
-  });
+    after(() => {
+      cy.task('esArchiverUnload', 'all_users');
+    });
 
-  after(() => {
-    cy.task('esArchiverUnload', 'all_users');
-  });
+    it('renders page as expected', () => {
+      cy.get(PAGE_TITLE).should('have.text', 'Entity Risk Score');
+    });
 
-  it('renders page as expected', () => {
-    cy.get(PAGE_TITLE).should('have.text', 'Entity Risk Score');
-  });
+    describe('Risk preview', () => {
+      it('risk scores reacts on change in datepicker', () => {
+        const START_DATE = 'Jan 18, 2019 @ 20:33:29.186';
+        const END_DATE = 'Jan 19, 2019 @ 20:33:29.186';
 
-  describe('Risk preview', () => {
-    it('risk scores reacts on change in datepicker', () => {
-      const START_DATE = 'Jan 18, 2019 @ 20:33:29.186';
-      const END_DATE = 'Jan 19, 2019 @ 20:33:29.186';
+        cy.get(HOST_RISK_PREVIEW_TABLE_ROWS).should('have.length', 5);
+        cy.get(USER_RISK_PREVIEW_TABLE_ROWS).should('have.length', 5);
 
-      cy.get(HOST_RISK_PREVIEW_TABLE_ROWS).should('have.length', 5);
-      cy.get(USER_RISK_PREVIEW_TABLE_ROWS).should('have.length', 5);
+        updateDateRangeInLocalDatePickers(LOCAL_QUERY_BAR_SELECTOR, START_DATE, END_DATE);
 
-      updateDateRangeInLocalDatePickers(LOCAL_QUERY_BAR_SELECTOR, START_DATE, END_DATE);
+        cy.get(HOST_RISK_PREVIEW_TABLE).contains('No items found');
+        cy.get(USER_RISK_PREVIEW_TABLE).contains('No items found');
+      });
 
-      cy.get(HOST_RISK_PREVIEW_TABLE).contains('No items found');
-      cy.get(USER_RISK_PREVIEW_TABLE).contains('No items found');
-    });
+      it('risk scores reacts on change in search bar query', () => {
+        cy.get(HOST_RISK_PREVIEW_TABLE_ROWS).should('have.length', 5);
+        cy.get(USER_RISK_PREVIEW_TABLE_ROWS).should('have.length', 5);
+
+        fillLocalSearchBar('host.name: "test-host1"');
+        submitLocalSearch(LOCAL_QUERY_BAR_SELECTOR);
+
+        cy.get(HOST_RISK_PREVIEW_TABLE_ROWS).should('have.length', 1);
+        cy.get(HOST_RISK_PREVIEW_TABLE_ROWS).contains('test-host1');
+        cy.get(USER_RISK_PREVIEW_TABLE_ROWS).should('have.length', 1);
+        cy.get(USER_RISK_PREVIEW_TABLE_ROWS).contains('test1');
+      });
+
+      it('show error panel if API returns error and then try to refetch data', () => {
+        interceptRiskPreviewError();
+
+        cy.get(RISK_PREVIEW_ERROR).contains('Preview failed');
 
-    it('risk scores reacts on change in search bar query', () => {
-      cy.get(HOST_RISK_PREVIEW_TABLE_ROWS).should('have.length', 5);
-      cy.get(USER_RISK_PREVIEW_TABLE_ROWS).should('have.length', 5);
+        interceptRiskPreviewSuccess();
 
-      fillLocalSearchBar('host.name: "test-host1"');
-      submitLocalSearch(LOCAL_QUERY_BAR_SELECTOR);
+        previewErrorButtonClick();
 
-      cy.get(HOST_RISK_PREVIEW_TABLE_ROWS).should('have.length', 1);
-      cy.get(HOST_RISK_PREVIEW_TABLE_ROWS).contains('test-host1');
-      cy.get(USER_RISK_PREVIEW_TABLE_ROWS).should('have.length', 1);
-      cy.get(USER_RISK_PREVIEW_TABLE_ROWS).contains('test1');
+        cy.get(RISK_PREVIEW_ERROR).should('not.exist');
+      });
     });
 
-    it('show error panel if API returns error and then try to refetch data', () => {
-      cy.intercept('POST', '/internal/risk_score/preview', {
-        statusCode: 500,
+    describe('Risk engine', () => {
+      it('should init, disable and enable risk engine', () => {
+        cy.get(RISK_SCORE_STATUS).should('have.text', 'Off');
+
+        // init
+        riskEngineStatusChange();
+
+        cy.get(RISK_SCORE_STATUS).should('have.text', 'On');
+
+        // disable
+        riskEngineStatusChange();
+
+        cy.get(RISK_SCORE_STATUS).should('have.text', 'Off');
+
+        // enable
+        riskEngineStatusChange();
+
+        cy.get(RISK_SCORE_STATUS).should('have.text', 'On');
       });
 
-      cy.get(RISK_PREVIEW_ERROR).contains('Preview failed');
+      it('should show error panel if API returns error ', () => {
+        cy.get(RISK_SCORE_STATUS).should('have.text', 'Off');
 
-      cy.intercept('POST', '/internal/risk_score/preview', {
-        statusCode: 200,
-        body: {
-          scores: { host: [], user: [] },
-        },
+        interceptRiskInitError();
+
+        // init
+        riskEngineStatusChange();
+
+        cy.get(RISK_SCORE_ERROR_PANEL).contains('Sorry, there was an error');
       });
 
-      cy.get(RISK_PREVIEW_ERROR_BUTTON).click();
+      it('should update if there legacy risk score installed', () => {
+        installRiskScoreModule();
+        visit(ENTITY_ANALYTICS_MANAGEMENT_URL);
+
+        cy.get(RISK_SCORE_STATUS).should('not.exist');
 
-      cy.get(RISK_PREVIEW_ERROR).should('not.exist');
+        updateRiskEngine();
+        updateRiskEngineConfirm();
+
+        cy.get(RISK_SCORE_STATUS).should('have.text', 'On');
+
+        deleteRiskScore({ riskScoreEntity: RiskScoreEntity.host, spaceId: 'default' });
+      });
     });
-  });
-});
+  }
+);