[Security Solution][Detection Engine] adds alert suppression to ES|QL rule type

packages/kbn-securitysolution-utils/src/esql/get_index_list_from_esql_query.test.ts

@@ -33,4 +33,9 @@ describe('getIndexListFromEsqlQuery', () => {
     getIndexPatternFromESQLQueryMock.mockReturnValue('test-1 , test-2 ');
     expect(getIndexListFromEsqlQuery('From  test-1, test-2 ')).toEqual(['test-1', 'test-2']);
   });
+
+  it('should return empty array when getIndexPatternFromESQLQuery throws error', () => {
+    getIndexPatternFromESQLQueryMock.mockReturnValue(new Error('Fail to parse'));
+    expect(getIndexListFromEsqlQuery('From  test-1 []')).toEqual([]);
+  });
 });

packages/kbn-securitysolution-utils/src/esql/get_index_list_from_esql_query.ts

@@ -11,9 +11,13 @@ import { getIndexPatternFromESQLQuery } from '@kbn/esql-utils';
  * parses ES|QL query and returns array of indices
  */
 export const getIndexListFromEsqlQuery = (query: string | undefined): string[] => {
-  const indexString = getIndexPatternFromESQLQuery(query);
+  try {
+    const indexString = getIndexPatternFromESQLQuery(query);
 
-  return getIndexListFromIndexString(indexString);
+    return getIndexListFromIndexString(indexString);
+  } catch (e) {
+    return [];
+  }
 };
 
 /**

x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.test.ts

@@ -1267,6 +1267,7 @@ describe('rules schema', () => {
     // behaviour common for multiple rule types
     const cases = [
       { ruleType: 'threat_match', ruleMock: getCreateThreatMatchRulesSchemaMock() },
+      { ruleType: 'esql', ruleMock: getCreateEsqlRulesSchemaMock() },
       { ruleType: 'query', ruleMock: getCreateRulesSchemaMock() },
       { ruleType: 'saved_query', ruleMock: getCreateSavedQueryRulesSchemaMock() },
       { ruleType: 'eql', ruleMock: getCreateEqlRuleSchemaMock() },

x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts

@@ -560,14 +560,19 @@ export const EsqlRuleRequiredFields = z.object({
   query: RuleQuery,
 });
 
+export type EsqlRuleOptionalFields = z.infer<typeof EsqlRuleOptionalFields>;
+export const EsqlRuleOptionalFields = z.object({
+  alert_suppression: AlertSuppression.optional(),
+});
+
 export type EsqlRulePatchFields = z.infer<typeof EsqlRulePatchFields>;
-export const EsqlRulePatchFields = EsqlRuleRequiredFields.partial();
+export const EsqlRulePatchFields = EsqlRuleOptionalFields.merge(EsqlRuleRequiredFields.partial());
 
 export type EsqlRuleResponseFields = z.infer<typeof EsqlRuleResponseFields>;
-export const EsqlRuleResponseFields = EsqlRuleRequiredFields;
+export const EsqlRuleResponseFields = EsqlRuleOptionalFields.merge(EsqlRuleRequiredFields);
 
 export type EsqlRuleCreateFields = z.infer<typeof EsqlRuleCreateFields>;
-export const EsqlRuleCreateFields = EsqlRuleRequiredFields;
+export const EsqlRuleCreateFields = EsqlRuleOptionalFields.merge(EsqlRuleRequiredFields);
 
 export type EsqlRule = z.infer<typeof EsqlRule>;
 export const EsqlRule = SharedResponseProps.merge(EsqlRuleResponseFields);

x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml

@@ -826,17 +826,26 @@ components:
         - language
         - query
 
+    EsqlRuleOptionalFields:
+      type: object
+      properties:
+        alert_suppression:
+          $ref: './common_attributes.schema.yaml#/components/schemas/AlertSuppression'
+
     EsqlRulePatchFields:
       allOf:
+        - $ref: '#/components/schemas/EsqlRuleOptionalFields'
         - $ref: '#/components/schemas/EsqlRuleRequiredFields'
           x-modify: partial
 
     EsqlRuleResponseFields:
       allOf:
+        - $ref: '#/components/schemas/EsqlRuleOptionalFields'
         - $ref: '#/components/schemas/EsqlRuleRequiredFields'
 
     EsqlRuleCreateFields:
       allOf:
+        - $ref: '#/components/schemas/EsqlRuleOptionalFields'
         - $ref: '#/components/schemas/EsqlRuleRequiredFields'
 
     EsqlRule:

x-pack/plugins/security_solution/common/detection_engine/constants.ts

@@ -41,6 +41,7 @@ export const MINIMUM_LICENSE_FOR_SUPPRESSION = 'platinum' as const;
 
 export const SUPPRESSIBLE_ALERT_RULES: Type[] = [
   'threshold',
+  'esql',
   'saved_query',
   'query',
   'new_terms',

x-pack/plugins/security_solution/common/detection_engine/utils.test.ts

@@ -229,6 +229,7 @@ describe('Alert Suppression Rules', () => {
   describe('isSuppressibleAlertRule', () => {
     test('should return true for a suppressible rule type', () => {
       // Rule types that support alert suppression:
+      expect(isSuppressibleAlertRule('esql')).toBe(true);
       expect(isSuppressibleAlertRule('threshold')).toBe(true);
       expect(isSuppressibleAlertRule('saved_query')).toBe(true);
       expect(isSuppressibleAlertRule('query')).toBe(true);
@@ -238,7 +239,6 @@ describe('Alert Suppression Rules', () => {
 
       // Rule types that don't support alert suppression:
       expect(isSuppressibleAlertRule('machine_learning')).toBe(false);
-      expect(isSuppressibleAlertRule('esql')).toBe(false);
     });
 
     test('should return false for an unknown rule type', () => {
@@ -266,6 +266,7 @@ describe('Alert Suppression Rules', () => {
   describe('isSuppressionRuleConfiguredWithDuration', () => {
     test('should return true for a suppressible rule type', () => {
       // Rule types that support alert suppression:
+      expect(isSuppressionRuleConfiguredWithDuration('esql')).toBe(true);
       expect(isSuppressionRuleConfiguredWithDuration('threshold')).toBe(true);
       expect(isSuppressionRuleConfiguredWithDuration('saved_query')).toBe(true);
       expect(isSuppressionRuleConfiguredWithDuration('query')).toBe(true);
@@ -275,7 +276,6 @@ describe('Alert Suppression Rules', () => {
 
       // Rule types that don't support alert suppression:
       expect(isSuppressionRuleConfiguredWithDuration('machine_learning')).toBe(false);
-      expect(isSuppressionRuleConfiguredWithDuration('esql')).toBe(false);
     });
 
     test('should return false for an unknown rule type', () => {
@@ -288,6 +288,7 @@ describe('Alert Suppression Rules', () => {
   describe('isSuppressionRuleConfiguredWithGroupBy', () => {
     test('should return true for a suppressible rule type with groupBy', () => {
       // Rule types that support alert suppression groupBy:
+      expect(isSuppressionRuleConfiguredWithGroupBy('esql')).toBe(true);
       expect(isSuppressionRuleConfiguredWithGroupBy('saved_query')).toBe(true);
       expect(isSuppressionRuleConfiguredWithGroupBy('query')).toBe(true);
       expect(isSuppressionRuleConfiguredWithGroupBy('threat_match')).toBe(true);
@@ -296,7 +297,6 @@ describe('Alert Suppression Rules', () => {
 
       // Rule types that don't support alert suppression:
       expect(isSuppressionRuleConfiguredWithGroupBy('machine_learning')).toBe(false);
-      expect(isSuppressionRuleConfiguredWithGroupBy('esql')).toBe(false);
     });
 
     test('should return false for a threshold rule type', () => {
@@ -314,6 +314,7 @@ describe('Alert Suppression Rules', () => {
   describe('isSuppressionRuleConfiguredWithMissingFields', () => {
     test('should return true for a suppressible rule type with missing fields', () => {
       // Rule types that support alert suppression groupBy:
+      expect(isSuppressionRuleConfiguredWithMissingFields('esql')).toBe(true);
       expect(isSuppressionRuleConfiguredWithMissingFields('saved_query')).toBe(true);
       expect(isSuppressionRuleConfiguredWithMissingFields('query')).toBe(true);
       expect(isSuppressionRuleConfiguredWithMissingFields('threat_match')).toBe(true);
@@ -322,7 +323,6 @@ describe('Alert Suppression Rules', () => {
 
       // Rule types that don't support alert suppression:
       expect(isSuppressionRuleConfiguredWithMissingFields('machine_learning')).toBe(false);
-      expect(isSuppressionRuleConfiguredWithMissingFields('esql')).toBe(false);
     });
 
     test('should return false for a threshold rule type', () => {

x-pack/plugins/security_solution/common/experimental_features.ts

@@ -180,6 +180,11 @@ export const allowedExperimentalValues = Object.freeze({
    */
   disableTimelineSaveTour: false,
 
+  /**
+   * Enables alerts suppression for ES|QL rules
+   */
+  alertSuppressionForEsqlRuleEnabled: false,
+
   /**
    * Enables the risk engine privileges route
    * and associated callout in the UI

x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.test.ts

@@ -5,7 +5,13 @@
  * 2.0.
  */
 
-import { computeHasMetadataOperator } from './esql_validator';
+import { parseEsqlQuery, computeHasMetadataOperator } from './esql_validator';
+
+import { computeIsESQLQueryAggregating } from '@kbn/securitysolution-utils';
+
+jest.mock('@kbn/securitysolution-utils', () => ({ computeIsESQLQueryAggregating: jest.fn() }));
+
+const computeIsESQLQueryAggregatingMock = computeIsESQLQueryAggregating as jest.Mock;
 
 describe('computeHasMetadataOperator', () => {
   it('should be false if query does not have operator', () => {
@@ -44,3 +50,37 @@ describe('computeHasMetadataOperator', () => {
     ).toBe(true);
   });
 });
+
+describe('parseEsqlQuery', () => {
+  it('returns isMissingMetadataOperator true when query is not aggregating and does not have metadata operator', () => {
+    computeIsESQLQueryAggregatingMock.mockReturnValueOnce(false);
+
+    expect(parseEsqlQuery('from test*')).toEqual({
+      isEsqlQueryAggregating: false,
+      isMissingMetadataOperator: true,
+    });
+  });
+
+  it('returns isMissingMetadataOperator false when query is not aggregating and has metadata operator', () => {
+    computeIsESQLQueryAggregatingMock.mockReturnValueOnce(false);
+
+    expect(parseEsqlQuery('from test* metadata _id')).toEqual({
+      isEsqlQueryAggregating: false,
+      isMissingMetadataOperator: false,
+    });
+  });
+
+  it('returns isMissingMetadataOperator false when query is aggregating', () => {
+    computeIsESQLQueryAggregatingMock.mockReturnValue(true);
+
+    expect(parseEsqlQuery('from test*')).toEqual({
+      isEsqlQueryAggregating: true,
+      isMissingMetadataOperator: false,
+    });
+
+    expect(parseEsqlQuery('from test* metadata _id')).toEqual({
+      isEsqlQueryAggregating: true,
+      isMissingMetadataOperator: false,
+    });
+  });
+});

x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.ts

@@ -6,11 +6,10 @@
  */
 
 import { isEmpty } from 'lodash';
-
+import type { QueryClient } from '@tanstack/react-query';
 import { computeIsESQLQueryAggregating } from '@kbn/securitysolution-utils';
 
 import { KibanaServices } from '../../../common/lib/kibana';
-import { securitySolutionQueryClient } from '../../../common/containers/query_client/query_client_provider';
 
 import type { ValidationError, ValidationFunc } from '../../../shared_imports';
 import { isEsqlRule } from '../../../../common/detection_engine/utils';
@@ -48,7 +47,7 @@ export const computeHasMetadataOperator = (esqlQuery: string) => {
 export const esqlValidator = async (
   ...args: Parameters<ValidationFunc>
 ): Promise<ValidationError<ERROR_CODES> | void | undefined> => {
-  const [{ value, formData }] = args;
+  const [{ value, formData, customData }] = args;
   const { query: queryValue } = value as FieldValueQueryBar;
   const query = queryValue.query as string;
   const { ruleType } = formData as DefineStepRule;
@@ -59,19 +58,19 @@ export const esqlValidator = async (
   }
 
   try {
-    const services = KibanaServices.get();
+    const queryClient = (customData.value as { queryClient: QueryClient | undefined })?.queryClient;
 
-    const isEsqlQueryAggregating = computeIsESQLQueryAggregating(query);
+    const services = KibanaServices.get();
+    const { isEsqlQueryAggregating, isMissingMetadataOperator } = parseEsqlQuery(query);
 
-    // non-aggregating query which does not have metadata, is not a valid one
-    if (!isEsqlQueryAggregating && !computeHasMetadataOperator(query)) {
+    if (isMissingMetadataOperator) {
       return {
         code: ERROR_CODES.ERR_MISSING_ID_FIELD_FROM_RESULT,
         message: i18n.ESQL_VALIDATION_MISSING_ID_IN_QUERY_ERROR,
       };
     }
 
-    const columns = await securitySolutionQueryClient.fetchQuery(
+    const columns = await queryClient?.fetchQuery(
       getEsqlQueryConfig({ esqlQuery: query, search: services.data.search.search })
     );
 
@@ -92,3 +91,17 @@ export const esqlValidator = async (
     return constructValidationError(error);
   }
 };
+
+/**
+ * check if esql query valid for Security rule:
+ * - if it's non aggregation query it must have metadata operator
+ */
+export const parseEsqlQuery = (query: string) => {
+  const isEsqlQueryAggregating = computeIsESQLQueryAggregating(query);
+
+  return {
+    isEsqlQueryAggregating,
+    // non-aggregating query which does not have [metadata], is not a valid one
+    isMissingMetadataOperator: !isEsqlQueryAggregating && !computeHasMetadataOperator(query),
+  };
+};

x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.test.tsx

@@ -575,7 +575,7 @@ describe('description_step', () => {
     });
 
     describe('alert suppression', () => {
-      const ruleTypesWithoutSuppression: Type[] = ['esql', 'machine_learning'];
+      const ruleTypesWithoutSuppression: Type[] = ['machine_learning'];
       const suppressionFields = {
         groupByDuration: {
           unit: 'm',

x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/index.tsx

@@ -42,7 +42,7 @@ import { useKibana } from '../../../../common/lib/kibana';
 import { useRuleIndices } from '../../../rule_management/logic/use_rule_indices';
 import { EsqlAutocomplete } from '../esql_autocomplete';
 import { MultiSelectFieldsAutocomplete } from '../multi_select_fields';
-import { useInvestigationFields } from '../../hooks/use_investigation_fields';
+import { useAllEsqlRuleFields } from '../../hooks';
 import { MaxSignals } from '../max_signals';
 
 const CommonUseField = getUseField({ component: Field });
@@ -133,10 +133,11 @@ const StepAboutRuleComponent: FC<StepAboutRuleProps> = ({
     [getFields]
   );
 
-  const { investigationFields, isLoading: isInvestigationFieldsLoading } = useInvestigationFields({
-    esqlQuery: isEsqlRuleValue ? esqlQuery : undefined,
-    indexPatternsFields: indexPattern.fields,
-  });
+  const { fields: investigationFields, isLoading: isInvestigationFieldsLoading } =
+    useAllEsqlRuleFields({
+      esqlQuery: isEsqlRuleValue ? esqlQuery : undefined,
+      indexPatternsFields: indexPattern.fields,
+    });
 
   return (
     <>

x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx

@@ -28,6 +28,7 @@ import type { FieldSpec } from '@kbn/data-views-plugin/common';
 import usePrevious from 'react-use/lib/usePrevious';
 import type { BrowserFields } from '@kbn/timelines-plugin/common';
 import type { Type } from '@kbn/securitysolution-io-ts-alerting-types';
+import { useQueryClient } from '@tanstack/react-query';
 
 import type { SavedQuery } from '@kbn/data-plugin/public';
 import type { DataViewBase } from '@kbn/es-query';
@@ -99,6 +100,7 @@ import { AlertSuppressionMissingFieldsStrategyEnum } from '../../../../../common
 import { DurationInput } from '../duration_input';
 import { MINIMUM_LICENSE_FOR_SUPPRESSION } from '../../../../../common/detection_engine/constants';
 import { useUpsellingMessage } from '../../../../common/hooks/use_upselling';
+import { useAllEsqlRuleFields } from '../../hooks';
 import { useAlertSuppression } from '../../../rule_management/logic/use_alert_suppression';
 import { RelatedIntegrations } from '../../../rule_creation/components/related_integrations';
 
@@ -191,6 +193,8 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
   thresholdFields,
   enableThresholdSuppression,
 }) => {
+  const queryClient = useQueryClient();
+
   const { isSuppressionEnabled: isAlertSuppressionEnabled } = useAlertSuppression(ruleType);
   const mlCapabilities = useMlCapabilities();
   const [openTimelineSearch, setOpenTimelineSearch] = useState(false);
@@ -453,6 +457,13 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
   );
 
   const [{ queryBar }] = useFormData<DefineStepRule>({ form, watch: ['queryBar'] });
+
+  const { fields: esqlSuppressionFields, isLoading: isEsqlSuppressionLoading } =
+    useAllEsqlRuleFields({
+      esqlQuery: isEsqlRule(ruleType) ? (queryBar?.query?.query as string) : undefined,
+      indexPatternsFields: indexPattern.fields,
+    });
+
   const areSuppressionFieldsDisabledBySequence =
     isEqlRule(ruleType) &&
     isEqlSequenceQuery(queryBar?.query?.query as string) &&
@@ -745,14 +756,15 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
         path="queryBar"
         config={esqlQueryBarConfig}
         component={QueryBarDefineRule}
+        validationData={{ queryClient }}
         componentProps={{
           ...queryBarProps,
           dataTestSubj: 'detectionEngineStepDefineRuleEsqlQueryBar',
           idAria: 'detectionEngineStepDefineRuleEsqlQueryBar',
         }}
       />
     ),
-    [queryBarProps, esqlQueryBarConfig]
+    [queryBarProps, esqlQueryBarConfig, queryClient]
   );
 
   const QueryBarMemo = useMemo(
@@ -1060,9 +1072,13 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
                 path="groupByFields"
                 component={MultiSelectFieldsAutocomplete}
                 componentProps={{
-                  browserFields: termsAggregationFields,
+                  browserFields: isEsqlRule(ruleType)
+                    ? esqlSuppressionFields
+                    : termsAggregationFields,
                   isDisabled:
-                    !isAlertSuppressionLicenseValid || areSuppressionFieldsDisabledBySequence,
+                    !isAlertSuppressionLicenseValid ||
+                    areSuppressionFieldsDisabledBySequence ||
+                    isEsqlSuppressionLoading,
                   disabledText: areSuppressionFieldsDisabledBySequence
                     ? i18n.EQL_SEQUENCE_SUPPRESSION_DISABLE_TOOLTIP
                     : alertSuppressionUpsellingMessage,