[Security Solution][Detection Engine] Adds support for suppressing EQ…

…L sequence alerts (elastic#189725) ## Summary Provide support for suppressing EQL sequence alerts. Co-authored-by: kibanamachine <[email protected]> Co-authored-by: Marshall Main <[email protected]>
hop-dev · Dec 5, 2024 · 945939e · 945939e
1 parent 0ff2e17
commit 945939e
Show file tree

Hide file tree

Showing 43 changed files with 2,371 additions and 324 deletions.
diff --git a/x-pack/plugins/rule_registry/common/schemas/8.17.0/index.ts b/x-pack/plugins/rule_registry/common/schemas/8.17.0/index.ts
@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { ALERT_SUPPRESSION_TERMS } from '@kbn/rule-data-utils';
+import { SearchTypes } from '@kbn/data-plugin/common';
+import { AlertWithCommonFields880 } from '../8.8.0';
+
+import { SuppressionFields8130 } from '../8.13.0';
+
+/* DO NOT MODIFY THIS SCHEMA TO ADD NEW FIELDS. These types represent the alerts that shipped in 8.13.0.
+Any changes to these types should be bug fixes so the types more accurately represent the alerts from 8.13.0.
+
+If you are adding new fields for a new release of Kibana, create a new sibling folder to this one
+for the version to be released and add the field(s) to the schema in that folder.
+
+Then, update `../index.ts` to import from the new folder that has the latest schemas, add the
+new schemas to the union of all alert schemas, and re-export the new schemas as the `*Latest` schemas.
+*/
+
+export interface SuppressionFields8170
+  extends Omit<SuppressionFields8130, typeof ALERT_SUPPRESSION_TERMS> {
+  [ALERT_SUPPRESSION_TERMS]: Array<{
+    field: string;
+    value: SearchTypes | null;
+  }>;
+}
+
+export type AlertWithSuppressionFields8170<T> = AlertWithCommonFields880<T> & SuppressionFields8170;
diff --git a/x-pack/plugins/rule_registry/common/schemas/index.ts b/x-pack/plugins/rule_registry/common/schemas/index.ts
@@ -13,11 +13,11 @@ import type {
   CommonAlertFields880,
 } from './8.8.0';
 
-import type { AlertWithSuppressionFields8130, SuppressionFields8130 } from './8.13.0';
+import type { AlertWithSuppressionFields8170, SuppressionFields8170 } from './8.17.0';
 
 export type {
-  AlertWithSuppressionFields8130 as AlertWithSuppressionFieldsLatest,
-  SuppressionFields8130 as SuppressionFieldsLatest,
+  AlertWithSuppressionFields8170 as AlertWithSuppressionFieldsLatest,
+  SuppressionFields8170 as SuppressionFieldsLatest,
   CommonAlertFieldName880 as CommonAlertFieldNameLatest,
   CommonAlertIdFieldName870 as CommonAlertIdFieldNameLatest,
   CommonAlertFields880 as CommonAlertFieldsLatest,

diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts
@@ -27,6 +27,7 @@ import {
   ALERT_RULE_EXECUTION_TIMESTAMP,
 } from '@kbn/rule-data-utils';
 import { mapKeys, snakeCase } from 'lodash/fp';
+
 import type { IRuleDataClient } from '..';
 import { getCommonAlertFields } from './get_common_alert_fields';
 import { CreatePersistenceRuleTypeWrapper } from './persistence_types';
@@ -471,9 +472,11 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper
                 }, {});
 
                 // filter out alerts that were already suppressed
-                // alert was suppressed if its suppression ends is older than suppression end of existing alert
-                // if existing alert was created earlier during the same rule execution - then alerts can be counted as not suppressed yet
-                // as they are processed for the first against this existing alert
+                // alert was suppressed if its suppression ends is older
+                // than suppression end of existing alert
+                // if existing alert was created earlier during the same
+                // rule execution - then alerts can be counted as not suppressed yet
+                // as they are processed for the first time against this existing alert
                 const nonSuppressedAlerts = filteredDuplicates.filter((alert) => {
                   const existingAlert =
                     existingAlertsByInstanceId[alert._source[ALERT_INSTANCE_ID]];
@@ -544,7 +547,15 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper
                   ];
                 });
 
-                let enrichedAlerts = newAlerts;
+                // we can now augment and enrich
+                // the sub alerts (if any) the same as we would
+                // any other newAlert
+                let enrichedAlerts = newAlerts.some((newAlert) => newAlert.subAlerts != null)
+                  ? newAlerts.flatMap((newAlert) => {
+                      const { subAlerts, ...everything } = newAlert;
+                      return [everything, ...(subAlerts ?? [])];
+                    })
+                  : newAlerts;
 
                 if (enrichAlerts) {
                   try {

diff --git a/x-pack/plugins/rule_registry/server/utils/persistence_types.ts b/x-pack/plugins/rule_registry/server/utils/persistence_types.ts
@@ -45,6 +45,10 @@ export type SuppressedAlertService = <T extends SuppressionFieldsLatest>(
   alerts: Array<{
     _id: string;
     _source: T;
+    subAlerts?: Array<{
+      _id: string;
+      _source: T;
+    }>;
   }>,
   suppressionWindow: string,
   enrichAlerts?: (

diff --git a/x-pack/plugins/security_solution/common/experimental_features.ts b/x-pack/plugins/security_solution/common/experimental_features.ts
@@ -12,6 +12,17 @@ export type ExperimentalFeatures = { [K in keyof typeof allowedExperimentalValue
  * This object is then used to validate and parse the value entered.
  */
 export const allowedExperimentalValues = Object.freeze({
+  /*
+   * Enables experimental feature flag for eql sequence alert suppression.
+   *
+   * Ticket: https://github.com/elastic/security-team/issues/9608
+   * Owners: https://github.com/orgs/elastic/teams/security-detection-engine
+   * Added: on October 1st, 2024 in https://github.com/elastic/kibana/pull/189725
+   * Turned: on (TBD)
+   * Expires: on (TBD)
+   */
+  alertSuppressionForSequenceEqlRuleEnabled: true,
+
   // FIXME:PT delete?
   excludePoliciesInFilterEnabled: false,
 

diff --git a/...y_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx b/...y_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx
@@ -145,7 +145,6 @@ const RuleTypeEuiFormRow = styled(EuiFormRow).attrs<{ $isVisible: boolean }>(({
   },
 }))<{ $isVisible: boolean }>``;
 
-// eslint-disable-next-line complexity
 const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
   dataSourceType,
   defaultSavedQuery,
@@ -172,7 +171,6 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
     watch: ['ruleType', 'queryBar', 'machineLearningJobId'],
   });
 
-  const { isSuppressionEnabled: isAlertSuppressionEnabled } = useAlertSuppression(ruleType);
   const [openTimelineSearch, setOpenTimelineSearch] = useState(false);
   const [indexModified, setIndexModified] = useState(false);
   const [threatIndexModified, setThreatIndexModified] = useState(false);
@@ -358,41 +356,31 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
    * purpose and so are treated as if the field is always selected.  */
   const areSuppressionFieldsSelected = isThresholdRule || Boolean(alertSuppressionFields?.length);
 
-  const areSuppressionFieldsDisabledBySequence =
-    isEqlRule(ruleType) &&
-    isEqlSequenceQuery(queryBar?.query?.query as string) &&
-    alertSuppressionFields?.length === 0;
+  const { isSuppressionEnabled: isAlertSuppressionEnabled } = useAlertSuppression(
+    isEqlSequenceQuery(queryBar?.query?.query as string)
+  );
 
   /** If we don't have ML field information, users can't meaningfully interact with suppression fields */
   const areSuppressionFieldsDisabledByMlFields =
     isMlRule(ruleType) && (mlRuleConfigLoading || !mlSuppressionFields.length);
 
   /** Suppression fields are generally disabled if either:
    * - License is insufficient (i.e. less than platinum)
-   * - An EQL Sequence is used
    * - ML Field information is not available
    */
   const areSuppressionFieldsDisabled =
-    !isAlertSuppressionLicenseValid ||
-    areSuppressionFieldsDisabledBySequence ||
-    areSuppressionFieldsDisabledByMlFields;
+    !isAlertSuppressionLicenseValid || areSuppressionFieldsDisabledByMlFields;
 
   const isSuppressionGroupByDisabled =
     (areSuppressionFieldsDisabled || isEsqlSuppressionLoading) && !areSuppressionFieldsSelected;
 
   const suppressionGroupByDisabledText = useMemo(() => {
-    if (areSuppressionFieldsDisabledBySequence) {
-      return i18n.EQL_SEQUENCE_SUPPRESSION_DISABLE_TOOLTIP;
-    } else if (areSuppressionFieldsDisabledByMlFields) {
+    if (areSuppressionFieldsDisabledByMlFields) {
       return i18n.MACHINE_LEARNING_SUPPRESSION_DISABLED_LABEL;
     } else {
       return alertSuppressionUpsellingMessage;
     }
-  }, [
-    alertSuppressionUpsellingMessage,
-    areSuppressionFieldsDisabledByMlFields,
-    areSuppressionFieldsDisabledBySequence,
-  ]);
+  }, [alertSuppressionUpsellingMessage, areSuppressionFieldsDisabledByMlFields]);
 
   const suppressionGroupByFields = useMemo(() => {
     if (isEsqlRule(ruleType)) {
@@ -824,13 +812,14 @@ const StepDefineRuleReadOnlyComponent: FC<StepDefineRuleReadOnlyProps> = ({
 }) => {
   const dataForDescription: Partial<DefineStepRule> = getStepDataDataSource(data);
   const transformFields = useExperimentalFeatureFieldsTransform();
+  const fieldsToDisplay = transformFields(dataForDescription);
 
   return (
     <StepContentWrapper data-test-subj="definitionRule" addPadding={addPadding}>
       <StepRuleDescription
         columns={descriptionColumns}
         schema={filterRuleFieldsForType(schema, data.ruleType)}
-        data={filterRuleFieldsForType(transformFields(dataForDescription), data.ruleType)}
+        data={filterRuleFieldsForType(fieldsToDisplay, data.ruleType)}
         indexPatterns={indexPattern}
       />
     </StepContentWrapper>

diff --git a/..._solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx b/..._solution/public/detection_engine/rule_creation_ui/components/step_define_rule/schema.tsx
@@ -16,8 +16,6 @@ import {
   customValidators,
 } from '../../../../common/components/threat_match/helpers';
 import {
-  isEqlRule,
-  isEqlSequenceQuery,
   isEsqlRule,
   isNewTermsRule,
   isThreatMatchRule,
@@ -44,7 +42,6 @@ import {
   THREAT_MATCH_INDEX_HELPER_TEXT,
   THREAT_MATCH_REQUIRED,
   THREAT_MATCH_EMPTIES,
-  EQL_SEQUENCE_SUPPRESSION_GROUPBY_VALIDATION_TEXT,
 } from './translations';
 import { queryRequiredValidatorFactory } from '../../validators/query_required_validator_factory';
 import { kueryValidatorFactory } from '../../validators/kuery_validator_factory';
@@ -587,33 +584,13 @@ export const schema: FormSchema<DefineStepRule> = {
         validator: (...args: Parameters<ValidationFunc>) => {
           const [{ formData }] = args;
           const needsValidation = isSuppressionRuleConfiguredWithGroupBy(formData.ruleType);
-
           if (!needsValidation) {
             return;
           }
 
           return alertSuppressionFieldsValidatorFactory()(...args);
         },
       },
-      {
-        validator: (
-          ...args: Parameters<ValidationFunc>
-        ): ReturnType<ValidationFunc<{}, ERROR_CODE>> | undefined => {
-          const [{ formData, value }] = args;
-
-          if (!isEqlRule(formData.ruleType) || !Array.isArray(value) || value.length === 0) {
-            return;
-          }
-
-          const query: string = formData.queryBar?.query?.query ?? '';
-
-          if (isEqlSequenceQuery(query)) {
-            return {
-              message: EQL_SEQUENCE_SUPPRESSION_GROUPBY_VALIDATION_TEXT,
-            };
-          }
-        },
-      },
     ],
   },
   [ALERT_SUPPRESSION_DURATION_FIELD_NAME]: {

diff --git a/...ion/public/detection_engine/rule_creation_ui/components/step_define_rule/translations.tsx b/...ion/public/detection_engine/rule_creation_ui/components/step_define_rule/translations.tsx
@@ -176,22 +176,6 @@ export const getEnableThresholdSuppressionLabel = (fields: string[] | undefined)
     )
   );
 
-export const EQL_SEQUENCE_SUPPRESSION_DISABLE_TOOLTIP = i18n.translate(
-  'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlSequenceSuppressionDisableText',
-  {
-    defaultMessage: 'Suppression is not supported for EQL sequence queries.',
-  }
-);
-
-export const EQL_SEQUENCE_SUPPRESSION_GROUPBY_VALIDATION_TEXT = i18n.translate(
-  'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.eqlSequenceSuppressionValidationText',
-  {
-    defaultMessage:
-      '{EQL_SEQUENCE_SUPPRESSION_DISABLE_TOOLTIP} Change the EQL query to a non-sequence query, or remove the suppression fields.',
-    values: { EQL_SEQUENCE_SUPPRESSION_DISABLE_TOOLTIP },
-  }
-);
-
 export const MACHINE_LEARNING_SUPPRESSION_DISABLED_LABEL = i18n.translate(
   'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.machineLearningSuppressionDisabledLabel',
   {

diff --git a/...rule_creation_ui/components/step_define_rule/use_experimental_feature_fields_transform.ts b/...rule_creation_ui/components/step_define_rule/use_experimental_feature_fields_transform.ts
@@ -7,16 +7,45 @@
 
 import { useCallback } from 'react';
 import type { DefineStepRule } from '../../../../detections/pages/detection_engine/rules/types';
+import { useIsExperimentalFeatureEnabled } from '../../../../common/hooks/use_experimental_features';
+import { isEqlRule, isEqlSequenceQuery } from '../../../../../common/detection_engine/utils';
+import {
+  ALERT_SUPPRESSION_FIELDS_FIELD_NAME,
+  ALERT_SUPPRESSION_DURATION_TYPE_FIELD_NAME,
+  ALERT_SUPPRESSION_DURATION_FIELD_NAME,
+  ALERT_SUPPRESSION_MISSING_FIELDS_FIELD_NAME,
+} from '../../../rule_creation/components/alert_suppression_edit';
 
 /**
  * transforms  DefineStepRule fields according to experimental feature flags
  */
 export const useExperimentalFeatureFieldsTransform = <T extends Partial<DefineStepRule>>(): ((
   fields: T
 ) => T) => {
-  const transformer = useCallback((fields: T) => {
-    return fields;
-  }, []);
+  const isAlertSuppressionForSequenceEqlRuleEnabled = useIsExperimentalFeatureEnabled(
+    'alertSuppressionForSequenceEqlRuleEnabled'
+  );
+  const transformer = useCallback(
+    (fields: T) => {
+      const isSuppressionDisabled =
+        isEqlRule(fields.ruleType) &&
+        isEqlSequenceQuery(fields.queryBar?.query?.query as string) &&
+        !isAlertSuppressionForSequenceEqlRuleEnabled;
+
+      // reset any alert suppression values hidden behind feature flag
+      if (isSuppressionDisabled) {
+        return {
+          ...fields,
+          [ALERT_SUPPRESSION_FIELDS_FIELD_NAME]: [],
+          [ALERT_SUPPRESSION_DURATION_TYPE_FIELD_NAME]: undefined,
+          [ALERT_SUPPRESSION_DURATION_FIELD_NAME]: undefined,
+          [ALERT_SUPPRESSION_MISSING_FIELDS_FIELD_NAME]: undefined,
+        };
+      }
+      return fields;
+    },
+    [isAlertSuppressionForSequenceEqlRuleEnabled]
+  );
 
   return transformer;
 };
diff --git a/...s/security_solution/public/detection_engine/rule_creation_ui/pages/rule_editing/index.tsx b/...s/security_solution/public/detection_engine/rule_creation_ui/pages/rule_editing/index.tsx
@@ -42,6 +42,7 @@ import { useUserData } from '../../../../detections/components/user_info';
 import { StepPanel } from '../../../rule_creation/components/step_panel';
 import { StepAboutRule } from '../../components/step_about_rule';
 import { StepDefineRule } from '../../components/step_define_rule';
+import { useExperimentalFeatureFieldsTransform } from '../../components/step_define_rule/use_experimental_feature_fields_transform';
 import { StepScheduleRule } from '../../components/step_schedule_rule';
 import { StepRuleActions } from '../../../rule_creation/components/step_rule_actions';
 import { formatRule } from '../rule_creation/helpers';
@@ -52,6 +53,7 @@ import {
   MaxWidthEuiFlexItem,
 } from '../../../../detections/pages/detection_engine/rules/helpers';
 import * as ruleI18n from '../../../../detections/pages/detection_engine/rules/translations';
+import type { DefineStepRule } from '../../../../detections/pages/detection_engine/rules/types';
 import { RuleStep } from '../../../../detections/pages/detection_engine/rules/types';
 import * as i18n from './translations';
 import { SecurityPageName } from '../../../../app/types';
@@ -368,11 +370,16 @@ const EditRulePageComponent: FC<{ rule: RuleResponse }> = ({ rule }) => {
 
   const { startTransaction } = useStartTransaction();
 
+  const defineFieldsTransform = useExperimentalFeatureFieldsTransform<DefineStepRule>();
+
   const saveChanges = useCallback(async () => {
     startTransaction({ name: SINGLE_RULE_ACTIONS.SAVE });
+    const localDefineStepData: DefineStepRule = defineFieldsTransform({
+      ...defineStepData,
+    });
     const updatedRule = await updateRule({
       ...formatRule<RuleUpdateProps>(
-        defineStepData,
+        localDefineStepData,
         aboutStepData,
         scheduleStepData,
         actionsStepData,
@@ -390,8 +397,9 @@ const EditRulePageComponent: FC<{ rule: RuleResponse }> = ({ rule }) => {
   }, [
     aboutStepData,
     actionsStepData,
-    addSuccess,
     defineStepData,
+    defineFieldsTransform,
+    addSuccess,
     navigateToApp,
     rule?.exceptions_list,
     ruleId,

diff --git a/...blic/detection_engine/rule_management/components/rule_details/rule_definition_section.tsx b/...blic/detection_engine/rule_management/components/rule_details/rule_definition_section.tsx
@@ -843,7 +843,7 @@ export const RuleDefinitionSection = ({
     ruleType: rule.type,
   });
 
-  const { isSuppressionEnabled } = useAlertSuppression(rule.type);
+  const { isSuppressionEnabled } = useAlertSuppression();
 
   const definitionSectionListItems = prepareDefinitionSectionListItems(
     rule,