From 6e6b99c4bb93feeebf4275075167abfbc0a6b39f Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Mon, 20 May 2024 12:16:39 +0100 Subject: [PATCH] [Security Solution][Detection Engine] adds alert suppression to ES|QL rule type (#180927) ## Summary - addresses https://github.com/elastic/security-team/issues/9203 - adds alert suppression for new terms rule type - similarly to [custom investigation fields](https://github.com/elastic/kibana/pull/177746) list of available suppression fields: - shows only ES|QL fields returned in query for aggregating queries - shows ES|QL fields returned in query + index fields for non-aggregating queries. Since resulted alerts for this type of query, are enriched with source documents. ### Demo 1. run esql rule w/o suppression 2. run esql rule w/ suppression per rule execution. Since ES|QL query is aggregating, no alerts suppressed on already agrregated field `host.ip` 3. run suppression on interval 20m 4. run suppression for custom ES|QL field which is the same as `host.ip`, hence same results 5. run suppression on interval 100m https://github.com/elastic/kibana/assets/92328789/4bd8cf13-6e23-4842-b775-605c74ae0127 ### Limitations Since suppressed alerts deduplication relies on alert timestamps, sorting of results other than `@timestamp asc` in ES|QL query may impact on number of suppressed alerts, when number of possible alerts more than max_signals. This affects only non-aggregating queries, since suppression boundaries for these alerts set as rule execution time ### Checklist - [x] Functional changes are hidden behind a feature flag Feature flag `alertSuppressionForEsqlRuleEnabled` - [x] Functional changes are covered with a test plan and automated tests. - https://github.com/elastic/security-team/pull/9389 - [x] Stability of new and changed tests is verified using the [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner). - FTR(x100): https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/5907 - Cypress(x100): https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/6011 - [x] Comprehensive manual testing is done by two engineers: the PR author and one of the PR reviewers. Changes are tested in both ESS and Serverless. - [x] Mapping changes are accompanied by a technical design document. It can be a GitHub issue or an RFC explaining the changes. The design document is shared with and approved by the appropriate teams and individual stakeholders. Existing AlertSuppression schema field is used for ES|QL rule, the one that already used for Query, New terms and IM rules. ```yml alert_suppression: $ref: './common_attributes.schema.yaml#/components/schemas/AlertSuppression' ``` where ```yml AlertSuppression: type: object properties: group_by: $ref: '#/components/schemas/AlertSuppressionGroupBy' duration: $ref: '#/components/schemas/AlertSuppressionDuration' missing_fields_strategy: $ref: '#/components/schemas/AlertSuppressionMissingFieldsStrategy' required: - group_by ``` - [x] Functional changes are communicated to the Docs team. A ticket or PR is opened in https://github.com/elastic/security-docs. The following information is included: any feature flags used, affected environments (Serverless, ESS, or both). - https://github.com/elastic/security-docs/issues/5156 --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Nikita Indik --- .../get_index_list_from_esql_query.test.ts | 5 + .../esql/get_index_list_from_esql_query.ts | 8 +- .../rule_schema/rule_request_schema.test.ts | 1 + .../model/rule_schema/rule_schemas.gen.ts | 11 +- .../rule_schema/rule_schemas.schema.yaml | 9 + .../common/detection_engine/constants.ts | 1 + .../common/detection_engine/utils.test.ts | 8 +- .../common/experimental_features.ts | 5 + .../installation_and_upgrade.md | 3 + .../logic/esql_validator.test.ts | 42 +- .../rule_creation/logic/esql_validator.ts | 29 +- .../description_step/index.test.tsx | 2 +- .../components/step_about_rule/index.tsx | 11 +- .../components/step_define_rule/index.tsx | 22 +- ...e_experimental_feature_fields_transform.ts | 29 +- .../rule_creation_ui/hooks/index.tsx | 1 + ...st.ts => use_all_esql_rule_fields.test.ts} | 113 +- .../hooks/use_all_esql_rule_fields.ts | 118 + .../hooks/use_investigation_fields.ts | 92 - .../pages/rule_creation/helpers.ts | 1 + .../logic/use_alert_suppression.test.tsx | 15 + .../logic/use_alert_suppression.tsx | 11 +- .../components/alerts_table/actions.tsx | 15 +- .../normalization/rule_converters.test.ts | 22 + .../normalization/rule_converters.ts | 4 + .../rule_schema/model/rule_schemas.mock.ts | 11 + .../rule_schema/model/rule_schemas.ts | 1 + .../rule_types/esql/create_esql_alert_type.ts | 4 +- .../detection_engine/rule_types/esql/esql.ts | 141 +- .../esql/utils/generate_alert_id.test.ts | 156 ++ .../esql/utils/generate_alert_id.ts | 57 + .../rule_types/esql/utils/index.ts | 1 + .../rule_types/esql/wrap_esql_alerts.test.ts | 94 + .../rule_types/esql/wrap_esql_alerts.ts | 47 +- .../esql/wrap_suppressed_esql_alerts.test.ts | 151 ++ .../esql/wrap_suppressed_esql_alerts.ts | 111 + ...bulk_create_suppressed_alerts_in_memory.ts | 2 +- ...bulk_create_suppressed_alerts_in_memory.ts | 13 +- .../partition_missing_fields_events.test.ts | 40 +- .../utils/partition_missing_fields_events.ts | 15 +- .../rule_types/utils/suppression_utils.ts | 5 +- .../config/ess/config.base.ts | 1 + .../configs/serverless.config.ts | 1 + .../execution_logic/esql_suppression.ts | 2025 +++++++++++++++++ .../execution_logic/index.ts | 1 + .../test/security_solution_cypress/config.ts | 1 + ...ws_suppression_serverless_essentials.cy.ts | 11 + .../common_flows_supression_ess_basic.cy.ts | 4 + .../rule_creation/esql_rule.cy.ts | 292 +++ .../rule_creation/esql_rule_ess.cy.ts | 219 -- .../rule_edit/esql_rule.cy.ts | 239 +- .../prebuilt_rules_preview.cy.ts | 66 +- .../cypress/objects/rule.ts | 2 +- .../cypress/tasks/create_new_rule.ts | 33 +- .../serverless_config.ts | 1 + 55 files changed, 3758 insertions(+), 565 deletions(-) rename x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/{use_investigation_fields.test.ts => use_all_esql_rule_fields.test.ts} (50%) create mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/use_all_esql_rule_fields.ts delete mode 100644 x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/use_investigation_fields.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/utils/generate_alert_id.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/utils/generate_alert_id.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_esql_alerts.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.ts create mode 100644 x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/esql_suppression.ts create mode 100644 x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/esql_rule.cy.ts delete mode 100644 x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/esql_rule_ess.cy.ts diff --git a/packages/kbn-securitysolution-utils/src/esql/get_index_list_from_esql_query.test.ts b/packages/kbn-securitysolution-utils/src/esql/get_index_list_from_esql_query.test.ts index ece5c72d5d5ff..5965e51e694e3 100644 --- a/packages/kbn-securitysolution-utils/src/esql/get_index_list_from_esql_query.test.ts +++ b/packages/kbn-securitysolution-utils/src/esql/get_index_list_from_esql_query.test.ts @@ -33,4 +33,9 @@ describe('getIndexListFromEsqlQuery', () => { getIndexPatternFromESQLQueryMock.mockReturnValue('test-1 , test-2 '); expect(getIndexListFromEsqlQuery('From test-1, test-2 ')).toEqual(['test-1', 'test-2']); }); + + it('should return empty array when getIndexPatternFromESQLQuery throws error', () => { + getIndexPatternFromESQLQueryMock.mockReturnValue(new Error('Fail to parse')); + expect(getIndexListFromEsqlQuery('From test-1 []')).toEqual([]); + }); }); diff --git a/packages/kbn-securitysolution-utils/src/esql/get_index_list_from_esql_query.ts b/packages/kbn-securitysolution-utils/src/esql/get_index_list_from_esql_query.ts index 64374732dc716..9464f041bdd64 100644 --- a/packages/kbn-securitysolution-utils/src/esql/get_index_list_from_esql_query.ts +++ b/packages/kbn-securitysolution-utils/src/esql/get_index_list_from_esql_query.ts @@ -11,9 +11,13 @@ import { getIndexPatternFromESQLQuery } from '@kbn/esql-utils'; * parses ES|QL query and returns array of indices */ export const getIndexListFromEsqlQuery = (query: string | undefined): string[] => { - const indexString = getIndexPatternFromESQLQuery(query); + try { + const indexString = getIndexPatternFromESQLQuery(query); - return getIndexListFromIndexString(indexString); + return getIndexListFromIndexString(indexString); + } catch (e) { + return []; + } }; /** diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.test.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.test.ts index 0b89cbdc72889..b7435c7dd86e8 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.test.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_request_schema.test.ts @@ -1267,6 +1267,7 @@ describe('rules schema', () => { // behaviour common for multiple rule types const cases = [ { ruleType: 'threat_match', ruleMock: getCreateThreatMatchRulesSchemaMock() }, + { ruleType: 'esql', ruleMock: getCreateEsqlRulesSchemaMock() }, { ruleType: 'query', ruleMock: getCreateRulesSchemaMock() }, { ruleType: 'saved_query', ruleMock: getCreateSavedQueryRulesSchemaMock() }, { ruleType: 'eql', ruleMock: getCreateEqlRuleSchemaMock() }, diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts index d2523a9a5c557..278b4679cd93e 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.gen.ts @@ -560,14 +560,19 @@ export const EsqlRuleRequiredFields = z.object({ query: RuleQuery, }); +export type EsqlRuleOptionalFields = z.infer; +export const EsqlRuleOptionalFields = z.object({ + alert_suppression: AlertSuppression.optional(), +}); + export type EsqlRulePatchFields = z.infer; -export const EsqlRulePatchFields = EsqlRuleRequiredFields.partial(); +export const EsqlRulePatchFields = EsqlRuleOptionalFields.merge(EsqlRuleRequiredFields.partial()); export type EsqlRuleResponseFields = z.infer; -export const EsqlRuleResponseFields = EsqlRuleRequiredFields; +export const EsqlRuleResponseFields = EsqlRuleOptionalFields.merge(EsqlRuleRequiredFields); export type EsqlRuleCreateFields = z.infer; -export const EsqlRuleCreateFields = EsqlRuleRequiredFields; +export const EsqlRuleCreateFields = EsqlRuleOptionalFields.merge(EsqlRuleRequiredFields); export type EsqlRule = z.infer; export const EsqlRule = SharedResponseProps.merge(EsqlRuleResponseFields); diff --git a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml index ae1a5657d2ab4..de424af505c1f 100644 --- a/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml +++ b/x-pack/plugins/security_solution/common/api/detection_engine/model/rule_schema/rule_schemas.schema.yaml @@ -826,17 +826,26 @@ components: - language - query + EsqlRuleOptionalFields: + type: object + properties: + alert_suppression: + $ref: './common_attributes.schema.yaml#/components/schemas/AlertSuppression' + EsqlRulePatchFields: allOf: + - $ref: '#/components/schemas/EsqlRuleOptionalFields' - $ref: '#/components/schemas/EsqlRuleRequiredFields' x-modify: partial EsqlRuleResponseFields: allOf: + - $ref: '#/components/schemas/EsqlRuleOptionalFields' - $ref: '#/components/schemas/EsqlRuleRequiredFields' EsqlRuleCreateFields: allOf: + - $ref: '#/components/schemas/EsqlRuleOptionalFields' - $ref: '#/components/schemas/EsqlRuleRequiredFields' EsqlRule: diff --git a/x-pack/plugins/security_solution/common/detection_engine/constants.ts b/x-pack/plugins/security_solution/common/detection_engine/constants.ts index b3bc1f434ff51..54c81cf93568f 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/constants.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/constants.ts @@ -41,6 +41,7 @@ export const MINIMUM_LICENSE_FOR_SUPPRESSION = 'platinum' as const; export const SUPPRESSIBLE_ALERT_RULES: Type[] = [ 'threshold', + 'esql', 'saved_query', 'query', 'new_terms', diff --git a/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts b/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts index e0c76253c416a..2e5ac39936fa3 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts @@ -229,6 +229,7 @@ describe('Alert Suppression Rules', () => { describe('isSuppressibleAlertRule', () => { test('should return true for a suppressible rule type', () => { // Rule types that support alert suppression: + expect(isSuppressibleAlertRule('esql')).toBe(true); expect(isSuppressibleAlertRule('threshold')).toBe(true); expect(isSuppressibleAlertRule('saved_query')).toBe(true); expect(isSuppressibleAlertRule('query')).toBe(true); @@ -238,7 +239,6 @@ describe('Alert Suppression Rules', () => { // Rule types that don't support alert suppression: expect(isSuppressibleAlertRule('machine_learning')).toBe(false); - expect(isSuppressibleAlertRule('esql')).toBe(false); }); test('should return false for an unknown rule type', () => { @@ -266,6 +266,7 @@ describe('Alert Suppression Rules', () => { describe('isSuppressionRuleConfiguredWithDuration', () => { test('should return true for a suppressible rule type', () => { // Rule types that support alert suppression: + expect(isSuppressionRuleConfiguredWithDuration('esql')).toBe(true); expect(isSuppressionRuleConfiguredWithDuration('threshold')).toBe(true); expect(isSuppressionRuleConfiguredWithDuration('saved_query')).toBe(true); expect(isSuppressionRuleConfiguredWithDuration('query')).toBe(true); @@ -275,7 +276,6 @@ describe('Alert Suppression Rules', () => { // Rule types that don't support alert suppression: expect(isSuppressionRuleConfiguredWithDuration('machine_learning')).toBe(false); - expect(isSuppressionRuleConfiguredWithDuration('esql')).toBe(false); }); test('should return false for an unknown rule type', () => { @@ -288,6 +288,7 @@ describe('Alert Suppression Rules', () => { describe('isSuppressionRuleConfiguredWithGroupBy', () => { test('should return true for a suppressible rule type with groupBy', () => { // Rule types that support alert suppression groupBy: + expect(isSuppressionRuleConfiguredWithGroupBy('esql')).toBe(true); expect(isSuppressionRuleConfiguredWithGroupBy('saved_query')).toBe(true); expect(isSuppressionRuleConfiguredWithGroupBy('query')).toBe(true); expect(isSuppressionRuleConfiguredWithGroupBy('threat_match')).toBe(true); @@ -296,7 +297,6 @@ describe('Alert Suppression Rules', () => { // Rule types that don't support alert suppression: expect(isSuppressionRuleConfiguredWithGroupBy('machine_learning')).toBe(false); - expect(isSuppressionRuleConfiguredWithGroupBy('esql')).toBe(false); }); test('should return false for a threshold rule type', () => { @@ -314,6 +314,7 @@ describe('Alert Suppression Rules', () => { describe('isSuppressionRuleConfiguredWithMissingFields', () => { test('should return true for a suppressible rule type with missing fields', () => { // Rule types that support alert suppression groupBy: + expect(isSuppressionRuleConfiguredWithMissingFields('esql')).toBe(true); expect(isSuppressionRuleConfiguredWithMissingFields('saved_query')).toBe(true); expect(isSuppressionRuleConfiguredWithMissingFields('query')).toBe(true); expect(isSuppressionRuleConfiguredWithMissingFields('threat_match')).toBe(true); @@ -322,7 +323,6 @@ describe('Alert Suppression Rules', () => { // Rule types that don't support alert suppression: expect(isSuppressionRuleConfiguredWithMissingFields('machine_learning')).toBe(false); - expect(isSuppressionRuleConfiguredWithMissingFields('esql')).toBe(false); }); test('should return false for a threshold rule type', () => { diff --git a/x-pack/plugins/security_solution/common/experimental_features.ts b/x-pack/plugins/security_solution/common/experimental_features.ts index da993a18debe3..565177fa8b560 100644 --- a/x-pack/plugins/security_solution/common/experimental_features.ts +++ b/x-pack/plugins/security_solution/common/experimental_features.ts @@ -180,6 +180,11 @@ export const allowedExperimentalValues = Object.freeze({ */ disableTimelineSaveTour: false, + /** + * Enables alerts suppression for ES|QL rules + */ + alertSuppressionForEsqlRuleEnabled: false, + /** * Enables the risk engine privileges route * and associated callout in the UI diff --git a/x-pack/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules/installation_and_upgrade.md b/x-pack/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules/installation_and_upgrade.md index 24aaf34764034..beda8a9517830 100644 --- a/x-pack/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules/installation_and_upgrade.md +++ b/x-pack/plugins/security_solution/docs/testing/test_plans/detection_response/prebuilt_rules/installation_and_upgrade.md @@ -198,6 +198,9 @@ Examples: │ New Terms │ Custom query │ Overview │ Definition │ │ New Terms │ Filters │ Overview │ Definition │ │ ESQL │ ESQL query │ Overview │ Definition │ +│ ESQL │ Suppress alerts by │ Overview │ Definition │ +│ ESQL │ Suppress alerts for │ Overview │ Definition │ +│ ESQL │ If a suppression field is missing │ Overview │ Definition │ ``` ## Scenarios diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.test.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.test.ts index 889d74a1c6503..07f14830d6a71 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.test.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.test.ts @@ -5,7 +5,13 @@ * 2.0. */ -import { computeHasMetadataOperator } from './esql_validator'; +import { parseEsqlQuery, computeHasMetadataOperator } from './esql_validator'; + +import { computeIsESQLQueryAggregating } from '@kbn/securitysolution-utils'; + +jest.mock('@kbn/securitysolution-utils', () => ({ computeIsESQLQueryAggregating: jest.fn() })); + +const computeIsESQLQueryAggregatingMock = computeIsESQLQueryAggregating as jest.Mock; describe('computeHasMetadataOperator', () => { it('should be false if query does not have operator', () => { @@ -44,3 +50,37 @@ describe('computeHasMetadataOperator', () => { ).toBe(true); }); }); + +describe('parseEsqlQuery', () => { + it('returns isMissingMetadataOperator true when query is not aggregating and does not have metadata operator', () => { + computeIsESQLQueryAggregatingMock.mockReturnValueOnce(false); + + expect(parseEsqlQuery('from test*')).toEqual({ + isEsqlQueryAggregating: false, + isMissingMetadataOperator: true, + }); + }); + + it('returns isMissingMetadataOperator false when query is not aggregating and has metadata operator', () => { + computeIsESQLQueryAggregatingMock.mockReturnValueOnce(false); + + expect(parseEsqlQuery('from test* metadata _id')).toEqual({ + isEsqlQueryAggregating: false, + isMissingMetadataOperator: false, + }); + }); + + it('returns isMissingMetadataOperator false when query is aggregating', () => { + computeIsESQLQueryAggregatingMock.mockReturnValue(true); + + expect(parseEsqlQuery('from test*')).toEqual({ + isEsqlQueryAggregating: true, + isMissingMetadataOperator: false, + }); + + expect(parseEsqlQuery('from test* metadata _id')).toEqual({ + isEsqlQueryAggregating: true, + isMissingMetadataOperator: false, + }); + }); +}); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.ts index b7e9f1b033e31..1f0bcb6596b40 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation/logic/esql_validator.ts @@ -6,11 +6,10 @@ */ import { isEmpty } from 'lodash'; - +import type { QueryClient } from '@tanstack/react-query'; import { computeIsESQLQueryAggregating } from '@kbn/securitysolution-utils'; import { KibanaServices } from '../../../common/lib/kibana'; -import { securitySolutionQueryClient } from '../../../common/containers/query_client/query_client_provider'; import type { ValidationError, ValidationFunc } from '../../../shared_imports'; import { isEsqlRule } from '../../../../common/detection_engine/utils'; @@ -48,7 +47,7 @@ export const computeHasMetadataOperator = (esqlQuery: string) => { export const esqlValidator = async ( ...args: Parameters ): Promise | void | undefined> => { - const [{ value, formData }] = args; + const [{ value, formData, customData }] = args; const { query: queryValue } = value as FieldValueQueryBar; const query = queryValue.query as string; const { ruleType } = formData as DefineStepRule; @@ -59,19 +58,19 @@ export const esqlValidator = async ( } try { - const services = KibanaServices.get(); + const queryClient = (customData.value as { queryClient: QueryClient | undefined })?.queryClient; - const isEsqlQueryAggregating = computeIsESQLQueryAggregating(query); + const services = KibanaServices.get(); + const { isEsqlQueryAggregating, isMissingMetadataOperator } = parseEsqlQuery(query); - // non-aggregating query which does not have metadata, is not a valid one - if (!isEsqlQueryAggregating && !computeHasMetadataOperator(query)) { + if (isMissingMetadataOperator) { return { code: ERROR_CODES.ERR_MISSING_ID_FIELD_FROM_RESULT, message: i18n.ESQL_VALIDATION_MISSING_ID_IN_QUERY_ERROR, }; } - const columns = await securitySolutionQueryClient.fetchQuery( + const columns = await queryClient?.fetchQuery( getEsqlQueryConfig({ esqlQuery: query, search: services.data.search.search }) ); @@ -92,3 +91,17 @@ export const esqlValidator = async ( return constructValidationError(error); } }; + +/** + * check if esql query valid for Security rule: + * - if it's non aggregation query it must have metadata operator + */ +export const parseEsqlQuery = (query: string) => { + const isEsqlQueryAggregating = computeIsESQLQueryAggregating(query); + + return { + isEsqlQueryAggregating, + // non-aggregating query which does not have [metadata], is not a valid one + isMissingMetadataOperator: !isEsqlQueryAggregating && !computeHasMetadataOperator(query), + }; +}; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.test.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.test.tsx index 7950493a1b989..8695041697120 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.test.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/description_step/index.test.tsx @@ -575,7 +575,7 @@ describe('description_step', () => { }); describe('alert suppression', () => { - const ruleTypesWithoutSuppression: Type[] = ['esql', 'machine_learning']; + const ruleTypesWithoutSuppression: Type[] = ['machine_learning']; const suppressionFields = { groupByDuration: { unit: 'm', diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/index.tsx index 3fa1852e6aa05..7666a9ba8aee3 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_about_rule/index.tsx @@ -42,7 +42,7 @@ import { useKibana } from '../../../../common/lib/kibana'; import { useRuleIndices } from '../../../rule_management/logic/use_rule_indices'; import { EsqlAutocomplete } from '../esql_autocomplete'; import { MultiSelectFieldsAutocomplete } from '../multi_select_fields'; -import { useInvestigationFields } from '../../hooks/use_investigation_fields'; +import { useAllEsqlRuleFields } from '../../hooks'; import { MaxSignals } from '../max_signals'; const CommonUseField = getUseField({ component: Field }); @@ -133,10 +133,11 @@ const StepAboutRuleComponent: FC = ({ [getFields] ); - const { investigationFields, isLoading: isInvestigationFieldsLoading } = useInvestigationFields({ - esqlQuery: isEsqlRuleValue ? esqlQuery : undefined, - indexPatternsFields: indexPattern.fields, - }); + const { fields: investigationFields, isLoading: isInvestigationFieldsLoading } = + useAllEsqlRuleFields({ + esqlQuery: isEsqlRuleValue ? esqlQuery : undefined, + indexPatternsFields: indexPattern.fields, + }); return ( <> diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx index 56073e2a6af59..839454922a14a 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/index.tsx @@ -28,6 +28,7 @@ import type { FieldSpec } from '@kbn/data-views-plugin/common'; import usePrevious from 'react-use/lib/usePrevious'; import type { BrowserFields } from '@kbn/timelines-plugin/common'; import type { Type } from '@kbn/securitysolution-io-ts-alerting-types'; +import { useQueryClient } from '@tanstack/react-query'; import type { SavedQuery } from '@kbn/data-plugin/public'; import type { DataViewBase } from '@kbn/es-query'; @@ -99,6 +100,7 @@ import { AlertSuppressionMissingFieldsStrategyEnum } from '../../../../../common import { DurationInput } from '../duration_input'; import { MINIMUM_LICENSE_FOR_SUPPRESSION } from '../../../../../common/detection_engine/constants'; import { useUpsellingMessage } from '../../../../common/hooks/use_upselling'; +import { useAllEsqlRuleFields } from '../../hooks'; import { useAlertSuppression } from '../../../rule_management/logic/use_alert_suppression'; import { RelatedIntegrations } from '../../../rule_creation/components/related_integrations'; @@ -191,6 +193,8 @@ const StepDefineRuleComponent: FC = ({ thresholdFields, enableThresholdSuppression, }) => { + const queryClient = useQueryClient(); + const { isSuppressionEnabled: isAlertSuppressionEnabled } = useAlertSuppression(ruleType); const mlCapabilities = useMlCapabilities(); const [openTimelineSearch, setOpenTimelineSearch] = useState(false); @@ -453,6 +457,13 @@ const StepDefineRuleComponent: FC = ({ ); const [{ queryBar }] = useFormData({ form, watch: ['queryBar'] }); + + const { fields: esqlSuppressionFields, isLoading: isEsqlSuppressionLoading } = + useAllEsqlRuleFields({ + esqlQuery: isEsqlRule(ruleType) ? (queryBar?.query?.query as string) : undefined, + indexPatternsFields: indexPattern.fields, + }); + const areSuppressionFieldsDisabledBySequence = isEqlRule(ruleType) && isEqlSequenceQuery(queryBar?.query?.query as string) && @@ -745,6 +756,7 @@ const StepDefineRuleComponent: FC = ({ path="queryBar" config={esqlQueryBarConfig} component={QueryBarDefineRule} + validationData={{ queryClient }} componentProps={{ ...queryBarProps, dataTestSubj: 'detectionEngineStepDefineRuleEsqlQueryBar', @@ -752,7 +764,7 @@ const StepDefineRuleComponent: FC = ({ }} /> ), - [queryBarProps, esqlQueryBarConfig] + [queryBarProps, esqlQueryBarConfig, queryClient] ); const QueryBarMemo = useMemo( @@ -1060,9 +1072,13 @@ const StepDefineRuleComponent: FC = ({ path="groupByFields" component={MultiSelectFieldsAutocomplete} componentProps={{ - browserFields: termsAggregationFields, + browserFields: isEsqlRule(ruleType) + ? esqlSuppressionFields + : termsAggregationFields, isDisabled: - !isAlertSuppressionLicenseValid || areSuppressionFieldsDisabledBySequence, + !isAlertSuppressionLicenseValid || + areSuppressionFieldsDisabledBySequence || + isEsqlSuppressionLoading, disabledText: areSuppressionFieldsDisabledBySequence ? i18n.EQL_SEQUENCE_SUPPRESSION_DISABLE_TOOLTIP : alertSuppressionUpsellingMessage, diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_experimental_feature_fields_transform.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_experimental_feature_fields_transform.ts index c035fef5af6e4..c92c35688dd3b 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_experimental_feature_fields_transform.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/components/step_define_rule/use_experimental_feature_fields_transform.ts @@ -7,6 +7,8 @@ import { useCallback } from 'react'; import type { DefineStepRule } from '../../../../detections/pages/detection_engine/rules/types'; +import { useIsExperimentalFeatureEnabled } from '../../../../common/hooks/use_experimental_features'; +import { isEsqlRule } from '../../../../../common/detection_engine/utils'; /** * transforms DefineStepRule fields according to experimental feature flags @@ -14,9 +16,30 @@ import type { DefineStepRule } from '../../../../detections/pages/detection_engi export const useExperimentalFeatureFieldsTransform = >(): (( fields: T ) => T) => { - const transformer = useCallback((fields: T) => { - return fields; - }, []); + const isAlertSuppressionForEsqlRuleEnabled = useIsExperimentalFeatureEnabled( + 'alertSuppressionForEsqlRuleEnabled' + ); + + const transformer = useCallback( + (fields: T) => { + const isSuppressionDisabled = + isEsqlRule(fields.ruleType) && !isAlertSuppressionForEsqlRuleEnabled; + + // reset any alert suppression values hidden behind feature flag + if (isSuppressionDisabled) { + return { + ...fields, + groupByFields: [], + groupByRadioSelection: undefined, + groupByDuration: undefined, + suppressionMissingFields: undefined, + }; + } + + return fields; + }, + [isAlertSuppressionForEsqlRuleEnabled] + ); return transformer; }; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/index.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/index.tsx index d56c317b93fb1..ea248587365aa 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/index.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/index.tsx @@ -7,3 +7,4 @@ export { useEsqlIndex } from './use_esql_index'; export { useEsqlQueryForAboutStep } from './use_esql_query_for_about_step'; +export { useAllEsqlRuleFields } from './use_all_esql_rule_fields'; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/use_investigation_fields.test.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/use_all_esql_rule_fields.test.ts similarity index 50% rename from x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/use_investigation_fields.test.ts rename to x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/use_all_esql_rule_fields.test.ts index 597d44f47f0d9..996b3ca044864 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/use_investigation_fields.test.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/use_all_esql_rule_fields.test.ts @@ -7,16 +7,15 @@ import { renderHook } from '@testing-library/react-hooks'; import type { DataViewFieldBase } from '@kbn/es-query'; +import { getESQLQueryColumns } from '@kbn/esql-utils'; -import { useInvestigationFields } from './use_investigation_fields'; +import { useAllEsqlRuleFields } from './use_all_esql_rule_fields'; import { createQueryWrapperMock } from '../../../common/__mocks__/query_wrapper'; +import { parseEsqlQuery } from '../../rule_creation/logic/esql_validator'; -import { computeIsESQLQueryAggregating } from '@kbn/securitysolution-utils'; -import { getESQLQueryColumns } from '@kbn/esql-utils'; - -jest.mock('@kbn/securitysolution-utils', () => ({ - computeIsESQLQueryAggregating: jest.fn(), +jest.mock('../../rule_creation/logic/esql_validator', () => ({ + parseEsqlQuery: jest.fn(), })); jest.mock('@kbn/esql-utils', () => { @@ -26,7 +25,7 @@ jest.mock('@kbn/esql-utils', () => { }; }); -const computeIsESQLQueryAggregatingMock = computeIsESQLQueryAggregating as jest.Mock; +const parseEsqlQueryMock = parseEsqlQuery as jest.Mock; const getESQLQueryColumnsMock = getESQLQueryColumns as jest.Mock; const { wrapper } = createQueryWrapperMock(); @@ -48,16 +47,26 @@ const mockEsqlDatatable = { columns: [{ id: '_custom_field', name: '_custom_field', meta: { type: 'string' } }], }; -describe('useInvestigationFields', () => { +describe('useAllEsqlRuleFields', () => { beforeEach(() => { jest.clearAllMocks(); - getESQLQueryColumnsMock.mockResolvedValue(mockEsqlDatatable.columns); + getESQLQueryColumnsMock.mockImplementation(({ esqlQuery }) => + Promise.resolve( + esqlQuery === 'deduplicate_test' + ? [ + { id: 'agent.name', name: 'agent.name', meta: { type: 'string' } }, // agent.name is already present in mockIndexPatternFields + { id: '_custom_field_0', name: '_custom_field_0', meta: { type: 'string' } }, + ] + : mockEsqlDatatable.columns + ) + ); + parseEsqlQueryMock.mockReturnValue({ isEsqlQueryAggregating: false }); }); it('should return loading true when esql fields still loading', () => { const { result } = renderHook( () => - useInvestigationFields({ + useAllEsqlRuleFields({ esqlQuery: mockEsqlQuery, indexPatternsFields: mockIndexPatternFields, }), @@ -68,71 +77,103 @@ describe('useInvestigationFields', () => { }); it('should return only index pattern fields when ES|QL query is empty', async () => { - const { result, waitForNextUpdate } = renderHook( + const { result } = renderHook( () => - useInvestigationFields({ + useAllEsqlRuleFields({ esqlQuery: '', indexPatternsFields: mockIndexPatternFields, }), { wrapper } ); - await waitForNextUpdate(); - - expect(result.current.investigationFields).toEqual(mockIndexPatternFields); + expect(result.current.fields).toEqual(mockIndexPatternFields); }); it('should return only index pattern fields when ES|QL query is undefined', async () => { const { result } = renderHook( () => - useInvestigationFields({ + useAllEsqlRuleFields({ esqlQuery: undefined, indexPatternsFields: mockIndexPatternFields, }), { wrapper } ); - expect(result.current.investigationFields).toEqual(mockIndexPatternFields); + expect(result.current.fields).toEqual(mockIndexPatternFields); }); it('should return index pattern fields concatenated with ES|QL fields when ES|QL query is non-aggregating', async () => { - computeIsESQLQueryAggregatingMock.mockReturnValue(false); + parseEsqlQueryMock.mockReturnValue({ isEsqlQueryAggregating: false }); - const { result } = renderHook( + const { result, waitFor } = renderHook( () => - useInvestigationFields({ + useAllEsqlRuleFields({ esqlQuery: mockEsqlQuery, indexPatternsFields: mockIndexPatternFields, }), { wrapper } ); - expect(result.current.investigationFields).toEqual([ - { - name: '_custom_field', - type: 'string', - }, - ...mockIndexPatternFields, - ]); + await waitFor(() => { + expect(result.current.fields).toEqual([ + { + name: '_custom_field', + type: 'string', + }, + ...mockIndexPatternFields, + ]); + }); }); it('should return only ES|QL fields when ES|QL query is aggregating', async () => { - computeIsESQLQueryAggregatingMock.mockReturnValue(true); + parseEsqlQueryMock.mockReturnValue({ isEsqlQueryAggregating: true }); - const { result } = renderHook( + const { result, waitFor } = renderHook( () => - useInvestigationFields({ + useAllEsqlRuleFields({ esqlQuery: mockEsqlQuery, indexPatternsFields: mockIndexPatternFields, }), { wrapper } ); + await waitFor(() => { + expect(result.current.fields).toEqual([ + { + name: '_custom_field', + type: 'string', + }, + ]); + }); + }); + + it('should deduplicate index pattern fields and ES|QL fields when fields have same name', async () => { + // getESQLQueryColumnsMock.mockClear(); + parseEsqlQueryMock.mockReturnValue({ isEsqlQueryAggregating: false }); + + const { result, waitFor } = renderHook( + () => + useAllEsqlRuleFields({ + esqlQuery: 'deduplicate_test', + indexPatternsFields: mockIndexPatternFields, + }), + { wrapper } + ); - expect(result.current.investigationFields).toEqual([ - { - name: '_custom_field', - type: 'string', - }, - ]); + await waitFor(() => { + expect(result.current.fields).toEqual([ + { + name: 'agent.name', + type: 'string', + }, + { + name: '_custom_field_0', + type: 'string', + }, + { + name: 'agent.type', + type: 'string', + }, + ]); + }); }); }); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/use_all_esql_rule_fields.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/use_all_esql_rule_fields.ts new file mode 100644 index 0000000000000..a67b990c88b80 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/use_all_esql_rule_fields.ts @@ -0,0 +1,118 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { useMemo, useState } from 'react'; +import type { DatatableColumn } from '@kbn/expressions-plugin/public'; +import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; +import type { DataViewFieldBase } from '@kbn/es-query'; +import useDebounce from 'react-use/lib/useDebounce'; + +import { useQuery } from '@tanstack/react-query'; + +import { useKibana } from '@kbn/kibana-react-plugin/public'; +import { parseEsqlQuery } from '../../rule_creation/logic/esql_validator'; + +import { getEsqlQueryConfig } from '../../rule_creation/logic/get_esql_query_config'; + +const esqlToFields = ( + columns: { error: unknown } | DatatableColumn[] | undefined | null +): DataViewFieldBase[] => { + if (columns && 'error' in columns) { + return []; + } + + const fields = (columns ?? []).map(({ id, meta }) => { + return { + name: id, + type: meta.type, + }; + }); + + return fields; +}; + +type UseEsqlFields = (esqlQuery: string | undefined) => { + isLoading: boolean; + fields: DataViewFieldBase[]; +}; + +/** + * fetches ES|QL fields and convert them to DataViewBase fields + */ +export const useEsqlFields: UseEsqlFields = (esqlQuery) => { + const kibana = useKibana<{ data: DataPublicPluginStart }>(); + + const { data: dataService } = kibana.services; + + const queryConfig = getEsqlQueryConfig({ esqlQuery, search: dataService?.search?.search }); + const { data, isLoading } = useQuery(queryConfig); + + const fields = useMemo(() => { + return esqlToFields(data); + }, [data]); + + return { + fields, + isLoading, + }; +}; + +/** + * if ES|QL fields and index pattern fields have same name, duplicates will be removed and the rest of fields merged + * ES|QL fields are first in order, since these are the fields that returned in ES|QL response + * */ +const deduplicateAndMergeFields = ( + esqlFields: DataViewFieldBase[], + indexPatternsFields: DataViewFieldBase[] +) => { + const esqlFieldsSet = new Set(esqlFields.map((field) => field.name)); + return [...esqlFields, ...indexPatternsFields.filter((field) => !esqlFieldsSet.has(field.name))]; +}; + +type UseAllEsqlRuleFields = (params: { + esqlQuery: string | undefined; + indexPatternsFields: DataViewFieldBase[]; +}) => { + isLoading: boolean; + fields: DataViewFieldBase[]; +}; + +/** + * returns all fields available for ES|QL rule: + * - fields returned from ES|QL query for aggregating queries + * - fields returned from ES|QL query + index fields for non-aggregating queries + */ +export const useAllEsqlRuleFields: UseAllEsqlRuleFields = ({ esqlQuery, indexPatternsFields }) => { + const [debouncedEsqlQuery, setDebouncedEsqlQuery] = useState(undefined); + const { fields: esqlFields, isLoading } = useEsqlFields(debouncedEsqlQuery); + + const { isEsqlQueryAggregating } = useMemo( + () => parseEsqlQuery(debouncedEsqlQuery ?? ''), + [debouncedEsqlQuery] + ); + + useDebounce( + () => { + setDebouncedEsqlQuery(esqlQuery); + }, + 300, + [esqlQuery] + ); + + const fields = useMemo(() => { + if (!debouncedEsqlQuery) { + return indexPatternsFields; + } + return isEsqlQueryAggregating + ? esqlFields + : deduplicateAndMergeFields(esqlFields, indexPatternsFields); + }, [esqlFields, debouncedEsqlQuery, indexPatternsFields, isEsqlQueryAggregating]); + + return { + fields, + isLoading, + }; +}; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/use_investigation_fields.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/use_investigation_fields.ts deleted file mode 100644 index 1627bddaa82be..0000000000000 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/hooks/use_investigation_fields.ts +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { useMemo } from 'react'; -import type { DatatableColumn } from '@kbn/expressions-plugin/public'; -import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; -import type { DataViewFieldBase } from '@kbn/es-query'; -import { computeIsESQLQueryAggregating } from '@kbn/securitysolution-utils'; - -import { useQuery } from '@tanstack/react-query'; - -import { useKibana } from '@kbn/kibana-react-plugin/public'; - -import { getEsqlQueryConfig } from '../../rule_creation/logic/get_esql_query_config'; - -const esqlToFields = ( - columns: { error: unknown } | DatatableColumn[] | undefined | null -): DataViewFieldBase[] => { - if (columns && 'error' in columns) { - return []; - } - - const fields = (columns ?? []).map(({ id, meta }) => { - return { - name: id, - type: meta.type, - }; - }); - - return fields; -}; - -type UseEsqlFields = (esqlQuery: string | undefined) => { - isLoading: boolean; - fields: DataViewFieldBase[]; -}; - -/** - * fetches ES|QL fields and convert them to DataViewBase fields - */ -const useEsqlFields: UseEsqlFields = (esqlQuery) => { - const kibana = useKibana<{ data: DataPublicPluginStart }>(); - - const { data: dataService } = kibana.services; - - const queryConfig = getEsqlQueryConfig({ esqlQuery, search: dataService?.search?.search }); - const { data, isLoading } = useQuery(queryConfig); - - const fields = useMemo(() => { - return esqlToFields(data); - }, [data]); - - return { - fields, - isLoading, - }; -}; - -type UseInvestigationFields = (params: { - esqlQuery: string | undefined; - indexPatternsFields: DataViewFieldBase[]; -}) => { - isLoading: boolean; - investigationFields: DataViewFieldBase[]; -}; - -export const useInvestigationFields: UseInvestigationFields = ({ - esqlQuery, - indexPatternsFields, -}) => { - const { fields: esqlFields, isLoading } = useEsqlFields(esqlQuery); - - const investigationFields = useMemo(() => { - if (!esqlQuery) { - return indexPatternsFields; - } - - // alerts generated from non-aggregating queries are enriched with source document - // so, index patterns fields should be included in the list of investigation fields - const isEsqlQueryAggregating = computeIsESQLQueryAggregating(esqlQuery); - - return isEsqlQueryAggregating ? esqlFields : [...esqlFields, ...indexPatternsFields]; - }, [esqlFields, esqlQuery, indexPatternsFields]); - - return { - investigationFields, - isLoading, - }; -}; diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.ts b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.ts index 11da431e3e602..f281b3b6b4a2b 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.ts +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_creation_ui/pages/rule_creation/helpers.ts @@ -512,6 +512,7 @@ export const formatDefineStepData = (defineStepData: DefineStepRule): DefineStep language: ruleFields.queryBar?.query?.language, query: ruleFields.queryBar?.query?.query as string, required_fields: requiredFields, + ...alertSuppressionFields, } : { ...alertSuppressionFields, diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.test.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.test.tsx index 0d9c0ebb75ca4..d12a5ff97d50a 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.test.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.test.tsx @@ -36,4 +36,19 @@ describe('useAlertSuppression', () => { expect(result.current.isSuppressionEnabled).toBe(false); }); + + it('should return isSuppressionEnabled false if ES|QL Feature Flag is disabled', () => { + const { result } = renderHook(() => useAlertSuppression('esql')); + + expect(result.current.isSuppressionEnabled).toBe(false); + }); + + it('should return isSuppressionEnabled true if ES|QL Feature Flag is enabled', () => { + jest + .spyOn(useIsExperimentalFeatureEnabledMock, 'useIsExperimentalFeatureEnabled') + .mockImplementation((flag) => flag === 'alertSuppressionForEsqlRuleEnabled'); + const { result } = renderHook(() => useAlertSuppression('esql')); + + expect(result.current.isSuppressionEnabled).toBe(true); + }); }); diff --git a/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.tsx b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.tsx index 6e1b2a4d6163f..1c9f139633c8c 100644 --- a/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.tsx +++ b/x-pack/plugins/security_solution/public/detection_engine/rule_management/logic/use_alert_suppression.tsx @@ -7,19 +7,28 @@ import { useCallback } from 'react'; import type { Type } from '@kbn/securitysolution-io-ts-alerting-types'; import { isSuppressibleAlertRule } from '../../../../common/detection_engine/utils'; +import { useIsExperimentalFeatureEnabled } from '../../../common/hooks/use_experimental_features'; export interface UseAlertSuppressionReturn { isSuppressionEnabled: boolean; } export const useAlertSuppression = (ruleType: Type | undefined): UseAlertSuppressionReturn => { + const isAlertSuppressionForEsqlRuleEnabled = useIsExperimentalFeatureEnabled( + 'alertSuppressionForEsqlRuleEnabled' + ); + const isSuppressionEnabledForRuleType = useCallback(() => { if (!ruleType) { return false; } + // Remove this condition when the Feature Flag for enabling Suppression in the New terms rule is removed. + if (ruleType === 'esql') { + return isSuppressibleAlertRule(ruleType) && isAlertSuppressionForEsqlRuleEnabled; + } return isSuppressibleAlertRule(ruleType); - }, [ruleType]); + }, [ruleType, isAlertSuppressionForEsqlRuleEnabled]); return { isSuppressionEnabled: isSuppressionEnabledForRuleType(), diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx b/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx index 3a8153d9fa2b3..7a3ed25b8084e 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx @@ -42,7 +42,7 @@ import { ALERT_NEW_TERMS, ALERT_RULE_INDICES, } from '../../../../common/field_maps/field_names'; -import { isEqlRule } from '../../../../common/detection_engine/utils'; +import { isEqlRule, isEsqlRule } from '../../../../common/detection_engine/utils'; import type { TimelineResult } from '../../../../common/api/timeline'; import { TimelineId } from '../../../../common/types/timeline'; import { TimelineStatus, TimelineType } from '../../../../common/api/timeline'; @@ -279,6 +279,11 @@ export const isEqlAlert = (ecsData: Ecs): boolean => { return isEqlRule(ruleType) || (Array.isArray(ruleType) && isEqlRule(ruleType[0])); }; +export const isEsqlAlert = (ecsData: Ecs): boolean => { + const ruleType = getField(ecsData, ALERT_RULE_TYPE); + return isEsqlRule(ruleType) || (Array.isArray(ruleType) && isEsqlRule(ruleType[0])); +}; + export const isNewTermsAlert = (ecsData: Ecs): boolean => { const ruleType = getField(ecsData, ALERT_RULE_TYPE); return ( @@ -1026,8 +1031,8 @@ export const sendAlertToTimelineAction = async ({ }, getExceptionFilter ); - // The Query field should remain unpopulated with the suppressed EQL alert. - } else if (isSuppressedAlert(ecsData) && !isEqlAlert(ecsData)) { + // The Query field should remain unpopulated with the suppressed EQL/ES|QL alert. + } else if (isSuppressedAlert(ecsData) && !isEqlAlert(ecsData) && !isEsqlAlert(ecsData)) { return createSuppressedTimeline( ecsData, createTimeline, @@ -1097,8 +1102,8 @@ export const sendAlertToTimelineAction = async ({ return createThresholdTimeline(ecsData, createTimeline, noteContent, {}, getExceptionFilter); } else if (isNewTermsAlert(ecsData)) { return createNewTermsTimeline(ecsData, createTimeline, noteContent, {}, getExceptionFilter); - // The Query field should remain unpopulated with the suppressed EQL alert. - } else if (isSuppressedAlert(ecsData) && !isEqlAlert(ecsData)) { + // The Query field should remain unpopulated with the suppressed EQL/ES|QL alert. + } else if (isSuppressedAlert(ecsData) && !isEqlAlert(ecsData) && !isEsqlAlert(ecsData)) { return createSuppressedTimeline(ecsData, createTimeline, noteContent, {}, getExceptionFilter); } else { let { dataProviders, filters } = buildTimelineDataProviderOrFilter( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.test.ts index d0a14151cabac..537d7b6abaf8a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.test.ts @@ -13,6 +13,7 @@ import { import { getBaseRuleParams, getEqlRuleParams, + getEsqlRuleParams, getMlRuleParams, getNewTermsRuleParams, getQueryRuleParams, @@ -219,6 +220,27 @@ describe('rule_converters', () => { ); }); + test('should accept ES|QL alerts suppression params', () => { + const patchParams = { + alert_suppression: { + group_by: ['agent.name'], + duration: { value: 4, unit: 'h' as const }, + missing_fields_strategy: 'doNotSuppress' as const, + }, + }; + const rule = getEsqlRuleParams(); + const patchedParams = patchTypeSpecificSnakeToCamel(patchParams, rule); + expect(patchedParams).toEqual( + expect.objectContaining({ + alertSuppression: { + groupBy: ['agent.name'], + missingFieldsStrategy: 'doNotSuppress', + duration: { value: 4, unit: 'h' }, + }, + }) + ); + }); + test('should accept threshold alerts suppression params', () => { const patchParams = { alert_suppression: { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts index 355fa626f7848..fd77213b178b5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/normalization/rule_converters.ts @@ -121,6 +121,7 @@ export const typeSpecificSnakeToCamel = ( type: params.type, language: params.language, query: params.query, + alertSuppression: convertAlertSuppressionToCamel(params.alert_suppression), }; } case 'threat_match': { @@ -237,6 +238,8 @@ const patchEsqlParams = ( type: existingRule.type, language: params.language ?? existingRule.language, query: params.query ?? existingRule.query, + alertSuppression: + convertAlertSuppressionToCamel(params.alert_suppression) ?? existingRule.alertSuppression, }; }; @@ -568,6 +571,7 @@ export const typeSpecificCamelToSnake = ( type: params.type, language: params.language, query: params.query, + alert_suppression: convertAlertSuppressionToSnake(params.alertSuppression), }; } case 'threat_match': { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.mock.ts index 9ebf17caf9a85..3a4fa1dadd778 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.mock.ts @@ -12,6 +12,7 @@ import type { BaseRuleParams, CompleteRule, EqlRuleParams, + EsqlRuleParams, MachineLearningRuleParams, NewTermsRuleParams, QueryRuleParams, @@ -103,6 +104,16 @@ export const getEqlRuleParams = (rewrites?: Partial): EqlRulePara }; }; +export const getEsqlRuleParams = (rewrites?: Partial): EsqlRuleParams => { + return { + ...getBaseRuleParams(), + type: 'esql', + language: 'esql', + query: 'from auditbeat* metadata _id', + ...rewrites, + }; +}; + export const getMlRuleParams = ( rewrites?: Partial ): MachineLearningRuleParams => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts index d116f6fd71209..120ddd3981165 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_schema/model/rule_schemas.ts @@ -169,6 +169,7 @@ export const EsqlSpecificRuleParams = z.object({ type: z.literal('esql'), language: z.literal('esql'), query: RuleQuery, + alertSuppression: AlertSuppressionCamel.optional(), }); export type EsqlRuleParams = BaseRuleParams & EsqlSpecificRuleParams; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/create_esql_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/create_esql_alert_type.ts index 15ff7adb11013..10c82ad8fed7c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/create_esql_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/create_esql_alert_type.ts @@ -16,7 +16,7 @@ import type { CreateRuleOptions, SecurityAlertType } from '../types'; export const createEsqlAlertType = ( createOptions: CreateRuleOptions ): SecurityAlertType => { - const { version } = createOptions; + const { version, experimentalFeatures, licensing } = createOptions; return { id: ESQL_RULE_TYPE_ID, name: 'ES|QL Rule', @@ -44,6 +44,6 @@ export const createEsqlAlertType = ( isExportable: false, category: DEFAULT_APP_CATEGORIES.security.id, producer: SERVER_APP_ID, - executor: (params) => esqlExecutor({ ...params, version }), + executor: (params) => esqlExecutor({ ...params, experimentalFeatures, version, licensing }), }; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql.ts index 64ed0560f3609..3887f5db81a5a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/esql.ts @@ -11,19 +11,24 @@ import type { AlertInstanceState, RuleExecutorServices, } from '@kbn/alerting-plugin/server'; +import type * as estypes from '@elastic/elasticsearch/lib/api/types'; import { computeIsESQLQueryAggregating, getIndexListFromEsqlQuery, } from '@kbn/securitysolution-utils'; +import type { LicensingPluginSetup } from '@kbn/licensing-plugin/server'; import { buildEsqlSearchRequest } from './build_esql_search_request'; import { performEsqlRequest } from './esql_request'; import { wrapEsqlAlerts } from './wrap_esql_alerts'; +import { wrapSuppressedEsqlAlerts } from './wrap_suppressed_esql_alerts'; +import { bulkCreateSuppressedAlertsInMemory } from '../utils/bulk_create_suppressed_alerts_in_memory'; import { createEnrichEventsFunction } from '../utils/enrichments'; import { rowToDocument } from './utils'; import { fetchSourceDocuments } from './fetch_source_documents'; +import { buildReasonMessageForEsqlAlert } from '../utils/reason_formatters'; -import type { RunOpts } from '../types'; +import type { RunOpts, SignalSource } from '../types'; import { addToSearchAfterReturn, @@ -31,16 +36,12 @@ import { makeFloatString, getUnprocessedExceptionsWarnings, getMaxSignalsWarning, + getSuppressionMaxSignalsWarning, } from '../utils/utils'; import type { EsqlRuleParams } from '../../rule_schema'; import { withSecuritySpan } from '../../../../utils/with_security_span'; - -/** - * ES|QL returns results as a single page. max size of 10,000 - * while we try increase size of the request to catch all events - * we don't want to overload ES/Kibana with large responses - */ -const ESQL_PAGE_SIZE_CIRCUIT_BREAKER = 1000; +import { getIsAlertSuppressionActive } from '../utils/get_is_alert_suppression_active'; +import type { ExperimentalFeatures } from '../../../../../common'; export const esqlExecutor = async ({ runOpts: { @@ -55,18 +56,29 @@ export const esqlExecutor = async ({ unprocessedExceptions, alertTimestampOverride, publicBaseUrl, + alertWithSuppression, }, services, state, spaceId, + experimentalFeatures, + licensing, }: { runOpts: RunOpts; services: RuleExecutorServices; state: object; spaceId: string; version: string; + experimentalFeatures: ExperimentalFeatures; + licensing: LicensingPluginSetup; }) => { const ruleParams = completeRule.ruleParams; + /** + * ES|QL returns results as a single page. max size of 10,000 + * while we try increase size of the request to catch all alerts that might been deduplicated + * we don't want to overload ES/Kibana with large responses + */ + const ESQL_PAGE_SIZE_CIRCUIT_BREAKER = tuple.maxSignals * 3; return withSecuritySpan('esqlExecutor', async () => { const result = createSearchAfterReturnType(); @@ -120,35 +132,100 @@ export const esqlExecutor = async ({ isRuleAggregating, }); - const wrappedAlerts = wrapEsqlAlerts({ - sourceDocuments, - isRuleAggregating, - results, - spaceId, - completeRule, - mergeStrategy, - alertTimestampOverride, - ruleExecutionLogger, - publicBaseUrl, - tuple, + const isAlertSuppressionActive = await getIsAlertSuppressionActive({ + alertSuppression: completeRule.ruleParams.alertSuppression, + licensing, + isFeatureDisabled: !experimentalFeatures?.alertSuppressionForEsqlRuleEnabled, }); - const enrichAlerts = createEnrichEventsFunction({ - services, - logger: ruleExecutionLogger, + const wrapHits = (events: Array>) => + wrapEsqlAlerts({ + events, + spaceId, + completeRule, + mergeStrategy, + isRuleAggregating, + alertTimestampOverride, + ruleExecutionLogger, + publicBaseUrl, + tuple, + }); + + const syntheticHits: Array> = results.map((document) => { + const { _id, _version, _index, ...source } = document; + + return { + _source: source as SignalSource, + fields: _id ? sourceDocuments[_id]?.fields : {}, + _id: _id ?? '', + _index: _index ?? '', + }; }); - const bulkCreateResult = await bulkCreate( - wrappedAlerts, - tuple.maxSignals - result.createdSignalsCount, - enrichAlerts - ); - addToSearchAfterReturn({ current: result, next: bulkCreateResult }); - ruleExecutionLogger.debug(`Created ${bulkCreateResult.createdItemsCount} alerts`); + if (isAlertSuppressionActive) { + const wrapSuppressedHits = (events: Array>) => + wrapSuppressedEsqlAlerts({ + events, + spaceId, + completeRule, + mergeStrategy, + isRuleAggregating, + alertTimestampOverride, + ruleExecutionLogger, + publicBaseUrl, + primaryTimestamp, + secondaryTimestamp, + tuple, + }); + + const bulkCreateResult = await bulkCreateSuppressedAlertsInMemory({ + enrichedEvents: syntheticHits, + toReturn: result, + wrapHits, + bulkCreate, + services, + ruleExecutionLogger, + tuple, + alertSuppression: completeRule.ruleParams.alertSuppression, + wrapSuppressedHits, + alertTimestampOverride, + alertWithSuppression, + experimentalFeatures, + buildReasonMessage: buildReasonMessageForEsqlAlert, + mergeSourceAndFields: true, + // passing 1 here since ES|QL does not support pagination + maxNumberOfAlertsMultiplier: 1, + }); + + addToSearchAfterReturn({ current: result, next: bulkCreateResult }); + ruleExecutionLogger.debug( + `Created ${bulkCreateResult.createdItemsCount} alerts. Suppressed ${bulkCreateResult.suppressedItemsCount} alerts` + ); + + if (bulkCreateResult.alertsWereTruncated) { + result.warningMessages.push(getSuppressionMaxSignalsWarning()); + break; + } + } else { + const wrappedAlerts = wrapHits(syntheticHits); + + const enrichAlerts = createEnrichEventsFunction({ + services, + logger: ruleExecutionLogger, + }); + const bulkCreateResult = await bulkCreate( + wrappedAlerts, + tuple.maxSignals - result.createdSignalsCount, + enrichAlerts + ); - if (bulkCreateResult.alertsWereTruncated) { - result.warningMessages.push(getMaxSignalsWarning()); - break; + addToSearchAfterReturn({ current: result, next: bulkCreateResult }); + ruleExecutionLogger.debug(`Created ${bulkCreateResult.createdItemsCount} alerts`); + + if (bulkCreateResult.alertsWereTruncated) { + result.warningMessages.push(getMaxSignalsWarning()); + break; + } } // no more results will be found diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/utils/generate_alert_id.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/utils/generate_alert_id.test.ts new file mode 100644 index 0000000000000..b3bc78e6b9478 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/utils/generate_alert_id.test.ts @@ -0,0 +1,156 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { generateAlertId } from './generate_alert_id'; +import type * as estypes from '@elastic/elasticsearch/lib/api/types'; +import type { SignalSource } from '../../types'; +import type { CompleteRule, EsqlRuleParams } from '../../../rule_schema'; +import moment from 'moment'; +import { cloneDeep } from 'lodash'; + +const mockEvent: estypes.SearchHit = { + _id: 'test_id', + _version: 2, + _index: 'test_index', +}; + +const mockRule = { + alertId: 'test_alert_id', + ruleParams: { + query: 'from auditbeat*', + }, +} as CompleteRule; + +describe('generateAlertId', () => { + describe('aggregating query', () => { + const aggIdParams = { + event: mockEvent, + spaceId: 'default', + completeRule: mockRule, + tuple: { + to: moment('2010-10-20 04:43:12'), + from: moment('2010-10-20 04:10:12'), + maxSignals: 100, + }, + isRuleAggregating: true, + index: 10, + }; + + const id = generateAlertId(aggIdParams); + let modifiedIdParams: Parameters['0']; + + beforeEach(() => { + modifiedIdParams = cloneDeep(aggIdParams); + }); + + it('creates id dependant on time range tuple', () => { + modifiedIdParams.tuple.from = moment('2010-10-20 04:20:12'); + expect(id).not.toBe(generateAlertId(modifiedIdParams)); + }); + + it('creates id dependant on data row index', () => { + modifiedIdParams.index = 11; + expect(id).not.toBe(generateAlertId(modifiedIdParams)); + }); + + it('creates id dependant on spaceId', () => { + modifiedIdParams.spaceId = 'test-1'; + expect(id).not.toBe(generateAlertId(modifiedIdParams)); + }); + + it('creates id not dependant on event._id', () => { + modifiedIdParams.event._id = 'another-id'; + expect(id).toBe(generateAlertId(modifiedIdParams)); + }); + it('creates id not dependant on event._version', () => { + modifiedIdParams.event._version = 100; + expect(id).toBe(generateAlertId(modifiedIdParams)); + }); + it('creates id not dependant on event._index', () => { + modifiedIdParams.event._index = 'packetbeat-*'; + expect(id).toBe(generateAlertId(modifiedIdParams)); + }); + it('creates id dependant on rule alertId', () => { + modifiedIdParams.completeRule.alertId = 'another-alert-id'; + expect(id).not.toBe(generateAlertId(modifiedIdParams)); + }); + + it('creates id dependant on rule query', () => { + modifiedIdParams.completeRule.ruleParams.query = 'from packetbeat*'; + expect(id).not.toBe(generateAlertId(modifiedIdParams)); + }); + }); + + describe('non-aggregating query', () => { + const nonAggIdParams = { + event: mockEvent, + spaceId: 'default', + completeRule: mockRule, + tuple: { + to: moment('2010-10-20 04:43:12'), + from: moment('2010-10-20 04:10:12'), + maxSignals: 100, + }, + isRuleAggregating: false, + index: 10, + }; + + const id = generateAlertId(nonAggIdParams); + let modifiedIdParams: Parameters['0']; + + beforeEach(() => { + modifiedIdParams = cloneDeep(nonAggIdParams); + }); + + it('creates id not dependant on time range tuple', () => { + modifiedIdParams.tuple.from = moment('2010-10-20 04:20:12'); + expect(id).toBe(generateAlertId(modifiedIdParams)); + }); + + it('creates id not dependant on data row index', () => { + modifiedIdParams.index = 11; + expect(id).toBe(generateAlertId(modifiedIdParams)); + }); + + it('creates id dependant on spaceId', () => { + modifiedIdParams.spaceId = 'test-1'; + expect(id).not.toBe(generateAlertId(modifiedIdParams)); + }); + + it('creates id dependant on event._id', () => { + modifiedIdParams.event._id = 'another-id'; + expect(id).not.toBe(generateAlertId(modifiedIdParams)); + }); + it('creates id dependant on event._version', () => { + modifiedIdParams.event._version = 100; + expect(id).not.toBe(generateAlertId(modifiedIdParams)); + }); + it('creates id dependant on event._index', () => { + modifiedIdParams.event._index = 'packetbeat-*'; + expect(id).not.toBe(generateAlertId(modifiedIdParams)); + }); + it('creates id dependant on rule alertId', () => { + modifiedIdParams.completeRule.alertId = 'another-alert-id'; + expect(id).not.toBe(generateAlertId(modifiedIdParams)); + }); + + it('creates id not dependant on rule query', () => { + modifiedIdParams.completeRule.ruleParams.query = 'from packetbeat*'; + expect(id).toBe(generateAlertId(modifiedIdParams)); + }); + + it('creates id dependant on suppression terms', () => { + modifiedIdParams.suppressionTerms = [{ field: 'agent.name', value: ['test-1'] }]; + const id1 = generateAlertId(modifiedIdParams); + modifiedIdParams.suppressionTerms = [{ field: 'agent.name', value: ['test-2'] }]; + const id2 = generateAlertId(modifiedIdParams); + + expect(id).not.toBe(id1); + expect(id1).not.toBe(id2); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/utils/generate_alert_id.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/utils/generate_alert_id.ts new file mode 100644 index 0000000000000..0f549783f922e --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/utils/generate_alert_id.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import objectHash from 'object-hash'; +import type { Moment } from 'moment'; +import type * as estypes from '@elastic/elasticsearch/lib/api/types'; + +import type { CompleteRule, EsqlRuleParams } from '../../../rule_schema'; +import type { SignalSource } from '../../types'; +import type { SuppressionTerm } from '../../utils/suppression_utils'; +/** + * Generates id for ES|QL alert. + * Id is generated as hash of event properties and rule/space config identifiers. + * This would allow to deduplicate alerts, generated from the same event. + */ +export const generateAlertId = ({ + event, + spaceId, + completeRule, + tuple, + isRuleAggregating, + index, + suppressionTerms, +}: { + isRuleAggregating: boolean; + event: estypes.SearchHit; + spaceId: string | null | undefined; + completeRule: CompleteRule; + tuple: { + to: Moment; + from: Moment; + maxSignals: number; + }; + index: number; + suppressionTerms?: SuppressionTerm[]; +}) => { + const ruleRunId = tuple.from.toISOString() + tuple.to.toISOString(); + + return !isRuleAggregating && event._id + ? objectHash([ + event._id, + event._version, + event._index, + `${spaceId}:${completeRule.alertId}`, + ...(suppressionTerms ? [suppressionTerms] : []), + ]) + : objectHash([ + ruleRunId, + completeRule.ruleParams.query, + `${spaceId}:${completeRule.alertId}`, + index, + ]); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/utils/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/utils/index.ts index 061ddfda93174..4b2b842680e32 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/utils/index.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/utils/index.ts @@ -6,3 +6,4 @@ */ export * from './row_to_document'; +export * from './generate_alert_id'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_esql_alerts.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_esql_alerts.test.ts new file mode 100644 index 0000000000000..d54f91c088958 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_esql_alerts.test.ts @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { cloneDeep } from 'lodash'; +import moment from 'moment'; + +import { ALERT_UUID } from '@kbn/rule-data-utils'; +import { getCompleteRuleMock, getEsqlRuleParams } from '../../rule_schema/mocks'; +import { ruleExecutionLogMock } from '../../rule_monitoring/mocks'; +import { sampleDocNoSortIdWithTimestamp } from '../__mocks__/es_results'; +import { wrapEsqlAlerts } from './wrap_esql_alerts'; + +import * as esqlUtils from './utils/generate_alert_id'; + +const ruleExecutionLogger = ruleExecutionLogMock.forExecutors.create(); + +const docId = 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71'; +const publicBaseUrl = 'http://somekibanabaseurl.com'; + +const alertSuppression = { + groupBy: ['source.ip'], +}; + +const completeRule = getCompleteRuleMock(getEsqlRuleParams()); +completeRule.ruleParams.alertSuppression = alertSuppression; + +describe('wrapSuppressedEsqlAlerts', () => { + test('should create an alert with the correct _id from a document', () => { + const doc = sampleDocNoSortIdWithTimestamp(docId); + const alerts = wrapEsqlAlerts({ + events: [doc], + isRuleAggregating: false, + spaceId: 'default', + mergeStrategy: 'missingFields', + completeRule, + alertTimestampOverride: undefined, + ruleExecutionLogger, + publicBaseUrl, + tuple: { + to: moment('2010-10-20 04:43:12'), + from: moment('2010-10-20 04:43:12'), + maxSignals: 100, + }, + }); + + expect(alerts[0]._id).toEqual('ed7fbf575371c898e0f0aea48cdf0bf1865939a9'); + expect(alerts[0]._source[ALERT_UUID]).toEqual('ed7fbf575371c898e0f0aea48cdf0bf1865939a9'); + }); + + test('should call generateAlertId for alert id', () => { + jest.spyOn(esqlUtils, 'generateAlertId').mockReturnValueOnce('mocked-alert-id'); + const completeRuleCloned = cloneDeep(completeRule); + completeRuleCloned.ruleParams.alertSuppression = { + groupBy: ['someKey'], + }; + const doc = sampleDocNoSortIdWithTimestamp(docId); + const alerts = wrapEsqlAlerts({ + events: [doc], + spaceId: 'default', + isRuleAggregating: true, + mergeStrategy: 'missingFields', + completeRule: completeRuleCloned, + alertTimestampOverride: undefined, + ruleExecutionLogger, + publicBaseUrl, + tuple: { + to: moment('2010-10-20 04:43:12'), + from: moment('2010-10-20 04:43:12'), + maxSignals: 100, + }, + }); + + expect(alerts[0]._id).toEqual('mocked-alert-id'); + expect(alerts[0]._source[ALERT_UUID]).toEqual('mocked-alert-id'); + + expect(esqlUtils.generateAlertId).toHaveBeenCalledWith( + expect.objectContaining({ + completeRule: expect.any(Object), + event: expect.any(Object), + index: 0, + isRuleAggregating: true, + spaceId: 'default', + tuple: { + from: expect.any(Object), + maxSignals: 100, + to: expect.any(Object), + }, + }) + ); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_esql_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_esql_alerts.ts index 6d3493e974668..b0fa2fd6638fa 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_esql_alerts.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_esql_alerts.ts @@ -5,7 +5,6 @@ * 2.0. */ -import objectHash from 'object-hash'; import type { Moment } from 'moment'; import type * as estypes from '@elastic/elasticsearch/lib/api/types'; @@ -19,9 +18,10 @@ import { buildReasonMessageForNewTermsAlert } from '../utils/reason_formatters'; import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; import { buildBulkBody } from '../factories/utils/build_bulk_body'; import type { SignalSource } from '../types'; +import { generateAlertId } from './utils'; export const wrapEsqlAlerts = ({ - results, + events, spaceId, completeRule, mergeStrategy, @@ -29,12 +29,10 @@ export const wrapEsqlAlerts = ({ ruleExecutionLogger, publicBaseUrl, tuple, - sourceDocuments, isRuleAggregating, }: { isRuleAggregating: boolean; - sourceDocuments: Record; - results: Array>; + events: Array>; spaceId: string | null | undefined; completeRule: CompleteRule; mergeStrategy: ConfigType['alertMergeStrategy']; @@ -47,37 +45,20 @@ export const wrapEsqlAlerts = ({ maxSignals: number; }; }): Array> => { - const wrapped = results.map>((document, i) => { - const ruleRunId = tuple.from.toISOString() + tuple.to.toISOString(); - - // for aggregating rules when metadata _id is present, generate alert based on ES document event id - const id = - !isRuleAggregating && document._id - ? objectHash([ - document._id, - document._version, - document._index, - `${spaceId}:${completeRule.alertId}`, - ]) - : objectHash([ - ruleRunId, - completeRule.ruleParams.query, - `${spaceId}:${completeRule.alertId}`, - i, - ]); - - // metadata fields need to be excluded from source, otherwise alerts creation fails - const { _id, _version, _index, ...source } = document; + const wrapped = events.map>((event, i) => { + const id = generateAlertId({ + event, + spaceId, + completeRule, + tuple, + isRuleAggregating, + index: i, + }); const baseAlert: BaseFieldsLatest = buildBulkBody( spaceId, completeRule, - { - _source: source as SignalSource, - fields: _id ? sourceDocuments[_id]?.fields : undefined, - _id: _id ?? '', - _index: _index ?? '', - }, + event, mergeStrategy, [], true, @@ -91,7 +72,7 @@ export const wrapEsqlAlerts = ({ return { _id: id, - _index: _index ?? '', + _index: event._index ?? '', _source: { ...baseAlert, }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.test.ts new file mode 100644 index 0000000000000..0c3c910efa056 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.test.ts @@ -0,0 +1,151 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { cloneDeep } from 'lodash'; +import moment from 'moment'; + +import { + ALERT_URL, + ALERT_UUID, + ALERT_SUPPRESSION_DOCS_COUNT, + ALERT_INSTANCE_ID, + ALERT_SUPPRESSION_TERMS, + ALERT_SUPPRESSION_START, + ALERT_SUPPRESSION_END, +} from '@kbn/rule-data-utils'; +import { getCompleteRuleMock, getEsqlRuleParams } from '../../rule_schema/mocks'; +import { ruleExecutionLogMock } from '../../rule_monitoring/mocks'; +import { sampleDocNoSortIdWithTimestamp } from '../__mocks__/es_results'; +import { wrapSuppressedEsqlAlerts } from './wrap_suppressed_esql_alerts'; + +import * as esqlUtils from './utils/generate_alert_id'; + +const ruleExecutionLogger = ruleExecutionLogMock.forExecutors.create(); + +const docId = 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71'; +const publicBaseUrl = 'http://somekibanabaseurl.com'; + +const alertSuppression = { + groupBy: ['source.ip'], +}; + +const completeRule = getCompleteRuleMock(getEsqlRuleParams()); +completeRule.ruleParams.alertSuppression = alertSuppression; + +describe('wrapSuppressedEsqlAlerts', () => { + test('should create an alert with the correct _id from a document and suppression fields', () => { + const doc = sampleDocNoSortIdWithTimestamp(docId); + const alerts = wrapSuppressedEsqlAlerts({ + events: [doc], + isRuleAggregating: false, + spaceId: 'default', + mergeStrategy: 'missingFields', + completeRule, + alertTimestampOverride: undefined, + ruleExecutionLogger, + publicBaseUrl, + primaryTimestamp: '@timestamp', + tuple: { + to: moment('2010-10-20 04:43:12'), + from: moment('2010-10-20 04:43:12'), + maxSignals: 100, + }, + }); + + expect(alerts[0]._id).toEqual('d94fb11e6062d7dce881ea07d952a1280398663a'); + expect(alerts[0]._source[ALERT_UUID]).toEqual('d94fb11e6062d7dce881ea07d952a1280398663a'); + expect(alerts[0]._source[ALERT_URL]).toContain( + 'http://somekibanabaseurl.com/app/security/alerts/redirect/d94fb11e6062d7dce881ea07d952a1280398663a?index=.alerts-security.alerts-default' + ); + expect(alerts[0]._source[ALERT_SUPPRESSION_DOCS_COUNT]).toEqual(0); + expect(alerts[0]._source[ALERT_INSTANCE_ID]).toEqual( + '1bf77f90e72d76d9335ad0ce356340a3d9833f96' + ); + expect(alerts[0]._source[ALERT_SUPPRESSION_TERMS]).toEqual([ + { field: 'source.ip', value: ['127.0.0.1'] }, + ]); + expect(alerts[0]._source[ALERT_SUPPRESSION_START]).toBeDefined(); + expect(alerts[0]._source[ALERT_SUPPRESSION_END]).toBeDefined(); + }); + + test('should create an alert with a different _id if suppression field is different', () => { + const completeRuleCloned = cloneDeep(completeRule); + completeRuleCloned.ruleParams.alertSuppression = { + groupBy: ['someKey'], + }; + const doc = sampleDocNoSortIdWithTimestamp(docId); + const alerts = wrapSuppressedEsqlAlerts({ + events: [doc], + spaceId: 'default', + isRuleAggregating: true, + mergeStrategy: 'missingFields', + completeRule: completeRuleCloned, + alertTimestampOverride: undefined, + ruleExecutionLogger, + publicBaseUrl, + primaryTimestamp: '@timestamp', + tuple: { + to: moment('2010-10-20 04:43:12'), + from: moment('2010-10-20 04:43:12'), + maxSignals: 100, + }, + }); + + expect(alerts[0]._source[ALERT_URL]).toContain( + 'http://somekibanabaseurl.com/app/security/alerts/redirect/' + ); + expect(alerts[0]._source[ALERT_SUPPRESSION_DOCS_COUNT]).toEqual(0); + expect(alerts[0]._source[ALERT_INSTANCE_ID]).toEqual( + 'c88edd552cb3501f040aea63ec68312e71af2ed2' + ); + expect(alerts[0]._source[ALERT_SUPPRESSION_TERMS]).toEqual([ + { field: 'someKey', value: 'someValue' }, + ]); + }); + + test('should call generateAlertId for alert id', () => { + jest.spyOn(esqlUtils, 'generateAlertId').mockReturnValueOnce('mocked-alert-id'); + const completeRuleCloned = cloneDeep(completeRule); + completeRuleCloned.ruleParams.alertSuppression = { + groupBy: ['someKey'], + }; + const doc = sampleDocNoSortIdWithTimestamp(docId); + const alerts = wrapSuppressedEsqlAlerts({ + events: [doc], + spaceId: 'default', + isRuleAggregating: false, + mergeStrategy: 'missingFields', + completeRule: completeRuleCloned, + alertTimestampOverride: undefined, + ruleExecutionLogger, + publicBaseUrl, + primaryTimestamp: '@timestamp', + tuple: { + to: moment('2010-10-20 04:43:12'), + from: moment('2010-10-20 04:43:12'), + maxSignals: 100, + }, + }); + + expect(alerts[0]._id).toEqual('mocked-alert-id'); + expect(alerts[0]._source[ALERT_UUID]).toEqual('mocked-alert-id'); + + expect(esqlUtils.generateAlertId).toHaveBeenCalledWith( + expect.objectContaining({ + completeRule: expect.any(Object), + event: expect.any(Object), + index: 0, + isRuleAggregating: false, + spaceId: 'default', + tuple: { + from: expect.any(Object), + maxSignals: 100, + to: expect.any(Object), + }, + }) + ); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.ts new file mode 100644 index 0000000000000..057cd5c906167 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/esql/wrap_suppressed_esql_alerts.ts @@ -0,0 +1,111 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import objectHash from 'object-hash'; +import type { Moment } from 'moment'; +import type * as estypes from '@elastic/elasticsearch/lib/api/types'; +import { TIMESTAMP } from '@kbn/rule-data-utils'; +import type { SuppressionFieldsLatest } from '@kbn/rule-registry-plugin/common/schemas'; + +import type { + BaseFieldsLatest, + WrappedFieldsLatest, +} from '../../../../../common/api/detection_engine/model/alerts'; +import type { ConfigType } from '../../../../config'; +import type { CompleteRule, EsqlRuleParams } from '../../rule_schema'; +import { buildReasonMessageForNewTermsAlert } from '../utils/reason_formatters'; +import type { IRuleExecutionLogForExecutors } from '../../rule_monitoring'; +import { buildBulkBody } from '../factories/utils/build_bulk_body'; +import type { SignalSource } from '../types'; +import { getSuppressionAlertFields, getSuppressionTerms } from '../utils'; +import { generateAlertId } from './utils'; + +export const wrapSuppressedEsqlAlerts = ({ + events, + spaceId, + completeRule, + mergeStrategy, + alertTimestampOverride, + ruleExecutionLogger, + publicBaseUrl, + tuple, + isRuleAggregating, + primaryTimestamp, + secondaryTimestamp, +}: { + isRuleAggregating: boolean; + events: Array>; + spaceId: string | null | undefined; + completeRule: CompleteRule; + mergeStrategy: ConfigType['alertMergeStrategy']; + alertTimestampOverride: Date | undefined; + ruleExecutionLogger: IRuleExecutionLogForExecutors; + publicBaseUrl: string | undefined; + tuple: { + to: Moment; + from: Moment; + maxSignals: number; + }; + primaryTimestamp: string; + secondaryTimestamp?: string; +}): Array> => { + const wrapped = events.map>( + (event, i) => { + const combinedFields = { ...event?.fields, ...event._source }; + + const suppressionTerms = getSuppressionTerms({ + alertSuppression: completeRule?.ruleParams?.alertSuppression, + fields: combinedFields, + }); + + const id = generateAlertId({ + event, + spaceId, + completeRule, + tuple, + isRuleAggregating, + index: i, + suppressionTerms, + }); + + const instanceId = objectHash([suppressionTerms, completeRule.alertId, spaceId]); + + const baseAlert: BaseFieldsLatest = buildBulkBody( + spaceId, + completeRule, + event, + mergeStrategy, + [], + true, + buildReasonMessageForNewTermsAlert, + [], + alertTimestampOverride, + ruleExecutionLogger, + id, + publicBaseUrl + ); + + return { + _id: id, + _index: event._index ?? '', + _source: { + ...baseAlert, + ...getSuppressionAlertFields({ + primaryTimestamp, + secondaryTimestamp, + fields: combinedFields, + suppressionTerms, + fallbackTimestamp: baseAlert[TIMESTAMP], + instanceId, + }), + }, + }; + } + ); + + return wrapped; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/bulk_create_suppressed_alerts_in_memory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/bulk_create_suppressed_alerts_in_memory.ts index 2fffa07f8d684..efa8e95c522a5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/bulk_create_suppressed_alerts_in_memory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/new_terms/bulk_create_suppressed_alerts_in_memory.ts @@ -87,7 +87,7 @@ export const bulkCreateSuppressedNewTermsAlertsInMemory = async ({ const partitionedEvents = partitionMissingFieldsEvents( eventsAndTerms, alertSuppression?.groupBy || [], - ['event'] + ['event', 'fields'] ); unsuppressibleWrappedDocs = wrapHits(partitionedEvents[1]); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts index 85bfc76e964e2..030cb213d94dd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/bulk_create_suppressed_alerts_in_memory.ts @@ -51,6 +51,8 @@ export interface BulkCreateSuppressedAlertsParams enrichedEvents: SignalSourceHit[]; toReturn: SearchAfterAndBulkCreateReturnType; experimentalFeatures: ExperimentalFeatures; + mergeSourceAndFields?: boolean; + maxNumberOfAlertsMultiplier?: number; } /** * wraps, bulk create and suppress alerts in memory, also takes care of missing fields logic. @@ -70,6 +72,8 @@ export const bulkCreateSuppressedAlertsInMemory = async ({ alertWithSuppression, alertTimestampOverride, experimentalFeatures, + mergeSourceAndFields = false, + maxNumberOfAlertsMultiplier, }: BulkCreateSuppressedAlertsParams) => { const suppressOnMissingFields = (alertSuppression?.missingFieldsStrategy ?? DEFAULT_SUPPRESSION_MISSING_FIELDS_STRATEGY) === @@ -81,7 +85,9 @@ export const bulkCreateSuppressedAlertsInMemory = async ({ if (!suppressOnMissingFields) { const partitionedEvents = partitionMissingFieldsEvents( enrichedEvents, - alertSuppression?.groupBy || [] + alertSuppression?.groupBy || [], + ['fields'], + mergeSourceAndFields ); unsuppressibleWrappedDocs = wrapHits(partitionedEvents[1], buildReasonMessage); @@ -102,6 +108,7 @@ export const bulkCreateSuppressedAlertsInMemory = async ({ alertWithSuppression, alertTimestampOverride, experimentalFeatures, + maxNumberOfAlertsMultiplier, }); }; @@ -120,6 +127,7 @@ export interface ExecuteBulkCreateAlertsParams>; toReturn: SearchAfterAndBulkCreateReturnType; experimentalFeatures: ExperimentalFeatures; + maxNumberOfAlertsMultiplier?: number; } /** @@ -139,11 +147,12 @@ export const executeBulkCreateAlerts = async < alertWithSuppression, alertTimestampOverride, experimentalFeatures, + maxNumberOfAlertsMultiplier = MAX_SIGNALS_SUPPRESSION_MULTIPLIER, }: ExecuteBulkCreateAlertsParams) => { // max signals for suppression includes suppressed and created alerts // this allows to lift max signals limitation to higher value // and can detects events beyond default max_signals value - const suppressionMaxSignals = MAX_SIGNALS_SUPPRESSION_MULTIPLIER * tuple.maxSignals; + const suppressionMaxSignals = maxNumberOfAlertsMultiplier * tuple.maxSignals; const suppressionDuration = alertSuppression?.duration; const suppressionWindow = suppressionDuration diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.test.ts index dfee32a058ba6..7fad1d4f2b10c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.test.ts @@ -30,7 +30,8 @@ describe('partitionMissingFieldsEvents', () => { _index: 'index-0', }, ], - ['agent.host', 'agent.type', 'agent.version'] + ['agent.host', 'agent.type', 'agent.version'], + ['fields'] ) ).toEqual([ [ @@ -83,7 +84,7 @@ describe('partitionMissingFieldsEvents', () => { }, ], ['agent.host', 'agent.type', 'agent.version'], - ['event'] + ['event', 'fields'] ) ).toEqual([ [ @@ -113,6 +114,35 @@ describe('partitionMissingFieldsEvents', () => { ], ]); }); + it('should partition when fields located in root of event', () => { + expect( + partitionMissingFieldsEvents( + [ + { + 'agent.host': 'host-1', + 'agent.version': 2, + }, + { + 'agent.host': 'host-1', + }, + ], + ['agent.host', 'agent.version'], + [] + ) + ).toEqual([ + [ + { + 'agent.host': 'host-1', + 'agent.version': 2, + }, + ], + [ + { + 'agent.host': 'host-1', + }, + ], + ]); + }); it('should partition if two fields are empty', () => { expect( partitionMissingFieldsEvents( @@ -125,7 +155,8 @@ describe('partitionMissingFieldsEvents', () => { _index: 'index-0', }, ], - ['agent.host', 'agent.type', 'agent.version'] + ['agent.host', 'agent.type', 'agent.version'], + ['fields'] ) ).toEqual([ [], @@ -152,7 +183,8 @@ describe('partitionMissingFieldsEvents', () => { _index: 'index-0', }, ], - ['agent.host', 'agent.type', 'agent.version'] + ['agent.host', 'agent.type', 'agent.version'], + ['fields'] ) ).toEqual([ [], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.ts index f86a969dc60c3..901768fe5c773 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/partition_missing_fields_events.ts @@ -17,20 +17,25 @@ import type { SignalSourceHit } from '../types'; * 2. where any of fields is empty */ export const partitionMissingFieldsEvents = < - T extends SignalSourceHit | { event: SignalSourceHit } + T extends SignalSourceHit | { event: SignalSourceHit } | Record >( events: T[], suppressedBy: string[] = [], // path to fields property within event object. At this point, it can be in root of event object or within event key - fieldsPath: ['event'] | [] = [] + fieldsPath: ['event', 'fields'] | ['fields'] | [] = [], + mergeSourceAndFields: boolean = false ): T[][] => { return partition(events, (event) => { if (suppressedBy.length === 0) { return true; } - const eventFields = get(event, [...fieldsPath, 'fields']); - const hasMissingFields = - Object.keys(pick(eventFields, suppressedBy)).length < suppressedBy.length; + const eventFields = fieldsPath.length ? get(event, fieldsPath) : event; + const sourceFields = + (event as SignalSourceHit)?._source || (event as { event: SignalSourceHit })?.event?._source; + + const fields = mergeSourceAndFields ? { ...sourceFields, ...eventFields } : eventFields; + + const hasMissingFields = Object.keys(pick(fields, suppressedBy)).length < suppressedBy.length; return !hasMissingFields; }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts index 167f22dc1d52b..44febba73e68e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/suppression_utils.ts @@ -17,7 +17,8 @@ import { ALERT_SUPPRESSION_END, } from '@kbn/rule-data-utils'; import type { AlertSuppressionCamel } from '../../../../../common/api/detection_engine/model/rule_schema'; -interface SuppressionTerm { + +export interface SuppressionTerm { field: string; value: string[] | number[] | null; } @@ -33,7 +34,7 @@ export const getSuppressionAlertFields = ({ fallbackTimestamp, instanceId, }: { - fields: Record | undefined; + fields: Record | undefined; primaryTimestamp: string; secondaryTimestamp?: string; suppressionTerms: SuppressionTerm[]; diff --git a/x-pack/test/security_solution_api_integration/config/ess/config.base.ts b/x-pack/test/security_solution_api_integration/config/ess/config.base.ts index cc47b97377e6c..d134546c6f633 100644 --- a/x-pack/test/security_solution_api_integration/config/ess/config.base.ts +++ b/x-pack/test/security_solution_api_integration/config/ess/config.base.ts @@ -80,6 +80,7 @@ export function createTestConfig(options: CreateTestConfigOptions, testFiles?: s '--xpack.ruleRegistry.unsafe.legacyMultiTenancy.enabled=true', `--xpack.securitySolution.enableExperimental=${JSON.stringify([ 'previewTelemetryUrlEnabled', + 'alertSuppressionForEsqlRuleEnabled', 'riskScoringPersistence', 'riskScoringRoutesEnabled', 'bulkCustomHighlightedFieldsEnabled', diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/configs/serverless.config.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/configs/serverless.config.ts index f6ba7fa49895e..76c73ff71cc18 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/configs/serverless.config.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/configs/serverless.config.ts @@ -19,6 +19,7 @@ export default createTestConfig({ ])}`, // See tests within the file "ignore_fields.ts" which use these values in "alertIgnoreFields" `--xpack.securitySolution.enableExperimental=${JSON.stringify([ 'bulkCustomHighlightedFieldsEnabled', + 'alertSuppressionForEsqlRuleEnabled', ])}`, ], }); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/esql_suppression.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/esql_suppression.ts new file mode 100644 index 0000000000000..f7264e064bdba --- /dev/null +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/esql_suppression.ts @@ -0,0 +1,2025 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import sortBy from 'lodash/sortBy'; +import expect from 'expect'; +import { v4 as uuidv4 } from 'uuid'; + +import { + ALERT_SUPPRESSION_START, + ALERT_SUPPRESSION_END, + ALERT_SUPPRESSION_DOCS_COUNT, + ALERT_SUPPRESSION_TERMS, + ALERT_LAST_DETECTED, + TIMESTAMP, + ALERT_START, +} from '@kbn/rule-data-utils'; +import { EsqlRuleCreateProps } from '@kbn/security-solution-plugin/common/api/detection_engine/model/rule_schema'; +import { getCreateEsqlRulesSchemaMock } from '@kbn/security-solution-plugin/common/api/detection_engine/model/rule_schema/mocks'; +import { RuleExecutionStatusEnum } from '@kbn/security-solution-plugin/common/api/detection_engine/rule_monitoring'; +import { ALERT_ORIGINAL_TIME } from '@kbn/security-solution-plugin/common/field_maps/field_names'; +import { DETECTION_ENGINE_SIGNALS_STATUS_URL as DETECTION_ENGINE_ALERTS_STATUS_URL } from '@kbn/security-solution-plugin/common/constants'; +import { getSuppressionMaxSignalsWarning as getSuppressionMaxAlertsWarning } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_types/utils/utils'; + +import { ENABLE_ASSET_CRITICALITY_SETTING } from '@kbn/security-solution-plugin/common/constants'; +import { + getPreviewAlerts, + previewRule, + getOpenAlerts, + dataGeneratorFactory, + previewRuleWithExceptionEntries, + setAlertStatus, + patchRule, +} from '../../../../utils'; +import { + deleteAllRules, + deleteAllAlerts, + createRule, +} from '../../../../../../../common/utils/security_solution'; +import { deleteAllExceptions } from '../../../../../lists_and_exception_lists/utils'; +import { FtrProviderContext } from '../../../../../../ftr_provider_context'; + +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + const es = getService('es'); + const log = getService('log'); + const kibanaServer = getService('kibanaServer'); + const { indexEnhancedDocuments, indexListOfDocuments, indexGeneratedDocuments } = + dataGeneratorFactory({ + es, + index: 'ecs_compliant', + log, + }); + + /** + * to separate docs between rules runs + */ + const internalIdPipe = (id: string) => `| where id=="${id}"`; + + const getNonAggRuleQueryWithMetadata = (id: string) => + `from ecs_compliant metadata _id, _index, _version ${internalIdPipe(id)}`; + + // skipped in MKI as it depends on feature flag alertSuppressionForEsqlRuleEnabled + describe('@ess @serverless @skipInServerlessMKI ES|QL rule type, alert suppression', () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/security_solution/ecs_compliant'); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/security_solution/ecs_compliant'); + await deleteAllAlerts(supertest, log, es); + await deleteAllRules(supertest, log); + }); + + it('should suppress an alert during real rule executions', async () => { + const id = uuidv4(); + const firstTimestamp = new Date().toISOString(); + + const firstExecutionDocuments = [ + { + host: { name: 'host-0' }, + id, + '@timestamp': firstTimestamp, + }, + { + host: { name: 'host-0' }, + id, + '@timestamp': firstTimestamp, + }, + { + host: { name: 'host-0' }, + id, + '@timestamp': firstTimestamp, + }, + ]; + + await indexListOfDocuments(firstExecutionDocuments); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: getNonAggRuleQueryWithMetadata(id), + from: 'now-35m', + interval: '30m', + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 300, + unit: 'm', + }, + missing_fields_strategy: 'suppress', + }, + }; + + const createdRule = await createRule(supertest, log, rule); + const alerts = await getOpenAlerts(supertest, log, es, createdRule); + + expect(alerts.hits.hits.length).toBe(1); + expect(alerts.hits.hits[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-0', + }, + ], + // suppression boundaries equal to original event time, since no alert been suppressed + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: firstTimestamp, + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + }) + ); + + const secondTimestamp = new Date().toISOString(); + const secondExecutionDocuments = [ + { + host: { name: 'host-0', ip: '127.0.0.5' }, + id, + '@timestamp': secondTimestamp, + }, + ]; + // Add a new document, then disable and re-enable to trigger another rule run. The second doc should + // trigger an update to the existing alert without changing the timestamp + await indexListOfDocuments(secondExecutionDocuments); + + await patchRule(supertest, log, { id: createdRule.id, enabled: false }); + await patchRule(supertest, log, { id: createdRule.id, enabled: true }); + const afterTimestamp = new Date(); + const secondAlerts = await getOpenAlerts( + supertest, + log, + es, + createdRule, + RuleExecutionStatusEnum.succeeded, + undefined, + afterTimestamp + ); + expect(secondAlerts.hits.hits.length).toEqual(1); + expect(secondAlerts.hits.hits[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-0', + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, // timestamp is the same + [ALERT_SUPPRESSION_START]: firstTimestamp, // suppression start is the same + [ALERT_SUPPRESSION_END]: secondTimestamp, // suppression end is updated + [ALERT_SUPPRESSION_DOCS_COUNT]: 3, + }) + ); + }); + + it('should NOT suppress and update an alert if the alert is closed', async () => { + const id = uuidv4(); + const firstTimestamp = new Date().toISOString(); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: getNonAggRuleQueryWithMetadata(id), + from: 'now-35m', + interval: '30m', + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 300, + unit: 'm', + }, + missing_fields_strategy: 'suppress', + }, + }; + + const firstExecutionDocuments = [ + { + host: { name: 'host-0' }, + id, + '@timestamp': firstTimestamp, + }, + ]; + + await indexListOfDocuments(firstExecutionDocuments); + + const createdRule = await createRule(supertest, log, rule); + const alerts = await getOpenAlerts(supertest, log, es, createdRule); + expect(alerts.hits.hits).toHaveLength(1); + // Close the alert. Subsequent rule executions should ignore this closed alert + // for suppression purposes. + const alertIds = alerts.hits.hits.map((alert) => alert._id); + await supertest + .post(DETECTION_ENGINE_ALERTS_STATUS_URL) + .set('kbn-xsrf', 'true') + .send(setAlertStatus({ alertIds, status: 'closed' })) + .expect(200); + + const secondTimestamp = new Date().toISOString(); + const secondExecutionDocuments = [ + { + host: { name: 'host-0' }, + id, + '@timestamp': secondTimestamp, + }, + ]; + // Add new documents, then disable and re-enable to trigger another rule run. The second doc should + // trigger a new alert since the first one is now closed. + await indexListOfDocuments(secondExecutionDocuments); + + await patchRule(supertest, log, { id: createdRule.id, enabled: false }); + await patchRule(supertest, log, { id: createdRule.id, enabled: true }); + const afterTimestamp = new Date(); + const secondAlerts = await getOpenAlerts( + supertest, + log, + es, + createdRule, + RuleExecutionStatusEnum.succeeded, + undefined, + afterTimestamp + ); + expect(secondAlerts.hits.hits.length).toEqual(1); + expect(alerts.hits.hits[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-0', + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }) + ); + }); + + it('should NOT suppress alerts when suppression period is less than rule interval', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:15:00.000Z'; + + const firstExecutionDocuments = [ + { + host: { name: 'host-0' }, + id, + '@timestamp': firstTimestamp, + }, + ]; + const secondExecutionDocuments = [ + { + host: { name: 'host-0' }, + id, + '@timestamp': secondTimestamp, + }, + ]; + await indexListOfDocuments([...firstExecutionDocuments, ...secondExecutionDocuments]); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: getNonAggRuleQueryWithMetadata(id), + from: 'now-35m', + interval: '30m', + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 10, + unit: 'm', + }, + missing_fields_strategy: 'suppress', + }, + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toBe(2); + expect(previewAlerts[0]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-0', + }, + ], + [TIMESTAMP]: '2020-10-28T06:00:00.000Z', + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: firstTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }) + ); + + expect(previewAlerts[1]._source).toEqual( + expect.objectContaining({ + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-0', + }, + ], + [TIMESTAMP]: '2020-10-28T06:30:00.000Z', + [ALERT_SUPPRESSION_START]: secondTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, + }) + ); + }); + + it('should suppress alerts in the time window that covers 3 rule executions', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:15:00.000Z'; + const thirdTimestamp = '2020-10-28T06:45:00.000Z'; + + const firstExecutionDocuments = [ + { + host: { name: 'host-0' }, + id, + '@timestamp': firstTimestamp, + }, + { + host: { name: 'host-0' }, + id, + '@timestamp': firstTimestamp, + }, + ]; + const secondExecutionDocuments = [ + { + host: { name: 'host-0' }, + id, + '@timestamp': secondTimestamp, + }, + ]; + const thirdExecutionDocuments = [ + { + host: { name: 'host-0' }, + id, + '@timestamp': thirdTimestamp, + }, + ]; + + await indexListOfDocuments([ + ...firstExecutionDocuments, + ...secondExecutionDocuments, + ...thirdExecutionDocuments, + ]); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: getNonAggRuleQueryWithMetadata(id), + from: 'now-35m', + interval: '30m', + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 2, + unit: 'h', + }, + missing_fields_strategy: 'suppress', + }, + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 3, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(1); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-0', + }, + ], + [TIMESTAMP]: '2020-10-28T06:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', // Note: ALERT_LAST_DETECTED gets updated, timestamp does not + [ALERT_START]: '2020-10-28T06:00:00.000Z', + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: thirdTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 3, // in total 3 alert got suppressed: 1 from the first run, 1 from the second, 1 from the third + }); + }); + + it('should suppress the correct alerts based on multi values group_by', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:15:00.000Z'; + + const firstExecutionDocuments = [ + { + host: { name: 'host-a' }, + id, + '@timestamp': firstTimestamp, + 'agent.version': 1, + }, + { + host: { name: 'host-a' }, + id, + '@timestamp': firstTimestamp, + 'agent.version': 2, + }, + { + host: { name: 'host-b' }, + id, + '@timestamp': firstTimestamp, + 'agent.version': 2, + }, + ]; + const secondExecutionDocuments = [ + { + host: { name: 'host-a' }, + id, + '@timestamp': secondTimestamp, + 'agent.version': 1, + }, + ]; + + await indexListOfDocuments([...firstExecutionDocuments, ...secondExecutionDocuments]); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: `from ecs_compliant metadata _id, _index, _version ${internalIdPipe( + id + )} | where host.name=="host-a"`, + from: 'now-35m', + interval: '30m', + alert_suppression: { + group_by: ['host.name', 'agent.version'], + duration: { + value: 60, + unit: 'm', + }, + missing_fields_strategy: 'suppress', + }, + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['host.name', 'agent.version', ALERT_ORIGINAL_TIME], + }); + // 3 alerts should be generated: + // 1. for pair 'host-a', 1 - suppressed + // 2. for pair 'host-a', 2 - not suppressed + expect(previewAlerts.length).toEqual(2); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + { + field: 'agent.version', + value: '1', + }, + ], + [TIMESTAMP]: '2020-10-28T06:00:00.000Z', + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + expect(previewAlerts[1]._source).toEqual({ + ...previewAlerts[1]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + { + field: 'agent.version', + value: '2', + }, + ], + [TIMESTAMP]: '2020-10-28T06:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T06:00:00.000Z', + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: firstTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 0, // no suppressed alerts + }); + }); + + it('should correctly suppress when using a timestamp override', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:10:00.000Z'; + + const docWithoutOverride = { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-a' }, + }; + const docWithOverride = { + ...docWithoutOverride, + host: { name: 'host-a' }, + // This simulates a very late arriving doc + '@timestamp': '2020-10-28T03:00:00.000Z', + event: { + ingested: secondTimestamp, + }, + }; + + await indexListOfDocuments([docWithoutOverride, docWithOverride]); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: getNonAggRuleQueryWithMetadata(id), + from: 'now-35m', + interval: '30m', + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 60, + unit: 'm', + }, + missing_fields_strategy: 'suppress', + }, + timestamp_override: 'event.ingested', + }; + + // 1 alert should be suppressed, based on event.ingested value of a document + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['host.name', ALERT_ORIGINAL_TIME], + }); + + expect(previewAlerts.length).toEqual(1); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + + it('should deduplicate multiple alerts while suppressing new ones', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:10:00.000Z'; + + const firstExecutionDocuments = [ + { + host: { name: 'host-a' }, + id, + '@timestamp': firstTimestamp, + }, + { + host: { name: 'host-a' }, + id, + '@timestamp': firstTimestamp, + }, + ]; + const secondExecutionDocuments = [ + { + host: { name: 'host-a' }, + id, + '@timestamp': secondTimestamp, + }, + { + host: { name: 'host-a' }, + id, + '@timestamp': secondTimestamp, + }, + { + host: { name: 'host-a' }, + id, + '@timestamp': secondTimestamp, + }, + ]; + + await indexListOfDocuments([...firstExecutionDocuments, ...secondExecutionDocuments]); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: getNonAggRuleQueryWithMetadata(id), + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 60, + unit: 'm', + }, + missing_fields_strategy: 'suppress', + }, + // large look-back time covers all docs + from: 'now-1h', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['host.name', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(1); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 4, + }); + }); + + it('should suppress alerts with missing fields', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:10:00.000Z'; + + const firstExecutionDocuments = [ + { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-a', ip: '127.0.0.3' }, + }, + { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-a', ip: '127.0.0.4' }, + }, + { + id, + '@timestamp': firstTimestamp, + host: { ip: '127.0.0.5' }, // doc 1 with missing host.name field + }, + { + id, + '@timestamp': firstTimestamp, + host: { ip: '127.0.0.6' }, // doc 2 with missing host.name field + }, + ]; + const secondExecutionDocuments = [ + { + host: { name: 'host-a', ip: '127.0.0.10' }, + id, + '@timestamp': secondTimestamp, + }, + { + host: { ip: '127.0.0.11' }, // doc 3 with missing host.name field + id, + '@timestamp': secondTimestamp, + }, + ]; + + await indexListOfDocuments([...firstExecutionDocuments, ...secondExecutionDocuments]); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: getNonAggRuleQueryWithMetadata(id), + from: 'now-35m', + interval: '30m', + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 60, + unit: 'm', + }, + missing_fields_strategy: 'suppress', + }, + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['host.name', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(2); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + }); + + expect(previewAlerts[1]._source).toEqual({ + ...previewAlerts[1]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: null, + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + }); + }); + + it('should not suppress alerts with missing fields if configured so', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:10:00.000Z'; + + const firstExecutionDocuments = [ + { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-a', ip: '127.0.0.3' }, + }, + { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-a', ip: '127.0.0.4' }, + }, + { + id, + '@timestamp': firstTimestamp, + host: { ip: '127.0.0.5' }, // doc 1 with missing host.name field + }, + { + id, + '@timestamp': firstTimestamp, + host: { ip: '127.0.0.6' }, // doc 2 with missing host.name field + }, + ]; + const secondExecutionDocuments = [ + { + host: { name: 'host-a', ip: '127.0.0.10' }, + id, + '@timestamp': secondTimestamp, + }, + { + host: { ip: '127.0.0.11' }, // doc 3 with missing host.name field + id, + '@timestamp': secondTimestamp, + }, + ]; + + await indexListOfDocuments([...firstExecutionDocuments, ...secondExecutionDocuments]); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: getNonAggRuleQueryWithMetadata(id), + from: 'now-35m', + interval: '30m', + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 60, + unit: 'm', + }, + missing_fields_strategy: 'doNotSuppress', + }, + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['host.name', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(4); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + }); + + // rest of alerts are not suppressed and do not have suppress properties + previewAlerts.slice(1).forEach((previewAlert) => { + const source = previewAlert._source; + expect(source).toHaveProperty('id', id); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_DOCS_COUNT); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_END); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_TERMS); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_DOCS_COUNT); + }); + }); + + it('should suppress alerts for aggregating queries', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:10:00.000Z'; + + const firstExecutionDocuments = [ + { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-a' }, + }, + { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-a' }, + }, + ]; + const secondExecutionDocuments = [ + { + host: { name: 'host-a' }, + id, + '@timestamp': secondTimestamp, + }, + { + host: { name: 'host-a' }, + id, + '@timestamp': secondTimestamp, + }, + { + host: { name: 'host-a' }, + id, + '@timestamp': secondTimestamp, + }, + ]; + + await indexListOfDocuments([...firstExecutionDocuments, ...secondExecutionDocuments]); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: `from ecs_compliant ${internalIdPipe( + id + )} | stats counted_agents=count(host.name) by host.name`, + from: 'now-35m', + interval: '30m', + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 60, + unit: 'm', + }, + missing_fields_strategy: 'suppress', + }, + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['host.name', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(1); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [ALERT_SUPPRESSION_START]: '2020-10-28T06:00:00.000Z', // since aggregation query results do not have timestamp properties suppression boundary start set as a first execution time + [ALERT_SUPPRESSION_END]: '2020-10-28T06:30:00.000Z', // since aggregation query results do not have timestamp properties suppression boundary end set as a second execution time + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, // only one suppressed alert, since aggregation query produces one alert per rule execution, no matter how many events aggregated + }); + expect(previewAlerts[0]._source).not.toHaveProperty(ALERT_ORIGINAL_TIME); + }); + + it('should suppress alerts by custom field, created in ES|QL query', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:10:00.000Z'; + + const firstExecutionDocuments = [ + { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-a' }, + }, + { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-b' }, + }, + ]; + const secondExecutionDocuments = [ + { + host: { name: 'test-c' }, + id, + '@timestamp': secondTimestamp, + }, + { + host: { name: 'host-d' }, + id, + '@timestamp': secondTimestamp, + }, + { + host: { name: 'test-s' }, + id, + '@timestamp': secondTimestamp, + }, + ]; + + await indexListOfDocuments([...firstExecutionDocuments, ...secondExecutionDocuments]); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + // ES|QL query creates new field custom_field - prefix_host, prefix_test + query: `from ecs_compliant metadata _id ${internalIdPipe( + id + )} | eval custom_field=concat("prefix_", left(host.name, 4))`, + from: 'now-35m', + interval: '30m', + alert_suppression: { + group_by: ['custom_field'], + duration: { + value: 60, + unit: 'm', + }, + missing_fields_strategy: 'suppress', + }, + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + }); + // lodash sortBy is used here because custom_field is non ECS and not mapped in alerts index, so can't be sorted by + const sortedAlerts = sortBy(previewAlerts, 'custom_field'); + expect(previewAlerts.length).toEqual(2); + + expect(sortedAlerts[0]._source).toEqual({ + ...sortedAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'custom_field', + value: 'prefix_host', + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + }); + + expect(sortedAlerts[1]._source).toEqual({ + ...sortedAlerts[1]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'custom_field', + value: 'prefix_test', + }, + ], + [ALERT_ORIGINAL_TIME]: secondTimestamp, + [ALERT_SUPPRESSION_START]: secondTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + + it('should suppress alerts by custom field, created in ES|QL query, when do not suppress missing fields configured', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:10:00.000Z'; + + const firstExecutionDocuments = [ + { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-a' }, + }, + { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-b' }, + }, + ]; + const secondExecutionDocuments = [ + { + host: { name: 'test-c' }, + id, + '@timestamp': secondTimestamp, + }, + { + host: { name: 'host-d' }, + id, + '@timestamp': secondTimestamp, + }, + { + host: { name: 'test-s' }, + id, + '@timestamp': secondTimestamp, + }, + ]; + + await indexListOfDocuments([...firstExecutionDocuments, ...secondExecutionDocuments]); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + // ES|QL query creates new field custom_field - prefix_host, prefix_test + query: `from ecs_compliant metadata _id ${internalIdPipe( + id + )} | eval custom_field=concat("prefix_", left(host.name, 4))`, + from: 'now-35m', + interval: '30m', + alert_suppression: { + group_by: ['custom_field'], + duration: { + value: 60, + unit: 'm', + }, + missing_fields_strategy: 'doNotSuppress', + }, + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + }); + // lodash sortBy is used here because custom_field is non ECS and not mapped in alerts index, so can't be sorted by + const sortedAlerts = sortBy(previewAlerts, 'custom_field'); + expect(previewAlerts.length).toEqual(2); + + expect(sortedAlerts[0]._source).toEqual({ + ...sortedAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'custom_field', + value: 'prefix_host', + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + }); + + expect(sortedAlerts[1]._source).toEqual({ + ...sortedAlerts[1]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'custom_field', + value: 'prefix_test', + }, + ], + [ALERT_ORIGINAL_TIME]: secondTimestamp, + [ALERT_SUPPRESSION_START]: secondTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + + it('should suppress by field, dropped in ES|QL query, but returned from source index', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:10:00.000Z'; + + const firstExecutionDocuments = [ + { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-a' }, + }, + ]; + const secondExecutionDocuments = [ + { + host: { name: 'host-a' }, + id, + '@timestamp': secondTimestamp, + }, + ]; + + await indexListOfDocuments([...firstExecutionDocuments, ...secondExecutionDocuments]); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: `from ecs_compliant metadata _id ${internalIdPipe(id)} | drop host.name`, + from: 'now-35m', + interval: '30m', + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 60, + unit: 'm', + }, + missing_fields_strategy: 'suppress', + }, + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['host.name', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(1); + + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a'], + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + + // even when field is dropped in ES|QL query it is returned from source document + it('should not suppress alerts by field, dropped in ES|QL query, when do not suppress missing fields configured', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:10:00.000Z'; + + const firstExecutionDocuments = [ + { + id, + '@timestamp': firstTimestamp, + host: { name: 'host-a' }, + }, + { + id, + '@timestamp': firstTimestamp, + }, + ]; + const secondExecutionDocuments = [ + { + host: { name: 'host-a' }, + id, + '@timestamp': secondTimestamp, + }, + { + id, + '@timestamp': firstTimestamp, + }, + ]; + + await indexListOfDocuments([...firstExecutionDocuments, ...secondExecutionDocuments]); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: `from ecs_compliant metadata _id ${internalIdPipe(id)} | drop host.name`, + from: 'now-35m', + interval: '30m', + alert_suppression: { + group_by: ['host.name'], + duration: { + value: 60, + unit: 'm', + }, + missing_fields_strategy: 'doNotSuppress', + }, + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['host.name', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(3); + + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a'], + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + + // rest of alerts are not suppressed and do not have suppress properties + previewAlerts.slice(1).forEach((previewAlert) => { + const source = previewAlert._source; + expect(source).toHaveProperty('id', id); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_DOCS_COUNT); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_END); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_TERMS); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_DOCS_COUNT); + }); + }); + + describe('rule execution only', () => { + it('should suppress alerts during rule execution only', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:45:00.000Z'; + const laterTimestamp = '2020-10-28T06:50:00.000Z'; + + const firstExecutionDocuments = [ + { + host: { name: 'host-a' }, + id, + '@timestamp': timestamp, + }, + { + host: { name: 'host-a' }, + id, + '@timestamp': timestamp, + }, + { + host: { name: 'host-a' }, + id, + '@timestamp': laterTimestamp, + }, + // does not generate alert + { + host: { name: 'host-b' }, + id, + '@timestamp': laterTimestamp, + }, + ]; + + await indexListOfDocuments(firstExecutionDocuments); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: `from ecs_compliant metadata _id, _index, _version ${internalIdPipe( + id + )} | where host.name=="host-a"`, + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(1); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_ORIGINAL_TIME]: timestamp, + [ALERT_SUPPRESSION_START]: timestamp, + [ALERT_SUPPRESSION_END]: laterTimestamp, // suppression ends with later timestamp + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + }); + }); + + it('should suppress alerts per rule execution for array field', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:45:00.000Z'; + + const firstExecutionDocuments = [ + { + host: { name: ['host-a', 'host-b'] }, + id, + '@timestamp': timestamp, + }, + { + host: { name: ['host-a', 'host-b'] }, + id, + '@timestamp': timestamp, + }, + ]; + + await indexListOfDocuments([...firstExecutionDocuments]); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: getNonAggRuleQueryWithMetadata(id), + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: [ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(1); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: ['host-a', 'host-b'], + }, + ], + [TIMESTAMP]: '2020-10-28T07:00:00.000Z', + [ALERT_LAST_DETECTED]: '2020-10-28T07:00:00.000Z', + [ALERT_ORIGINAL_TIME]: timestamp, + [ALERT_SUPPRESSION_START]: timestamp, + [ALERT_SUPPRESSION_END]: timestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + + it('should suppress alerts with missing fields during rule execution only for multiple suppress by fields', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:45:00.000Z'; + + const firstExecutionDocuments = [ + // no missing fields + { + host: { name: 'host-a', ip: '127.0.0.11' }, + agent: { name: 'agent-a', version: 10 }, + id, + '@timestamp': timestamp, + }, + { + host: { name: 'host-a', ip: '127.0.0.12' }, + agent: { name: 'agent-a', version: 10 }, + id, + '@timestamp': timestamp, + }, + // missing agent.name + { + host: { name: 'host-a', ip: '127.0.0.21' }, + agent: { version: 10 }, + id, + '@timestamp': timestamp, + }, + { + host: { name: 'host-a', ip: '127.0.0.22' }, + agent: { version: 10 }, + id, + '@timestamp': timestamp, + }, + // missing agent.version + { + host: { name: 'host-a', ip: '127.0.0.31' }, + agent: { name: 'agent-a' }, + id, + '@timestamp': timestamp, + }, + { + host: { name: 'host-a', ip: '127.0.0.32' }, + agent: { name: 'agent-a' }, + id, + '@timestamp': timestamp, + }, + // missing both agent.* + { + host: { name: 'host-a', ip: '127.0.0.41' }, + id, + '@timestamp': timestamp, + }, + { + host: { name: 'host-a', ip: '127.0.0.42' }, + id, + '@timestamp': timestamp, + }, + ]; + + await indexListOfDocuments(firstExecutionDocuments); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: getNonAggRuleQueryWithMetadata(id), + alert_suppression: { + group_by: ['agent.name', 'agent.version'], + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['agent.name', 'agent.version', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(4); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: 'agent-a', + }, + { + field: 'agent.version', + value: '10', + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + + expect(previewAlerts[1]._source).toEqual({ + ...previewAlerts[1]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: 'agent-a', + }, + { + field: 'agent.version', + value: null, + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + + expect(previewAlerts[2]._source).toEqual({ + ...previewAlerts[2]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: null, + }, + { + field: 'agent.version', + value: '10', + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + + expect(previewAlerts[3]._source).toEqual({ + ...previewAlerts[3]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: null, + }, + { + field: 'agent.version', + value: null, + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + + it('should not suppress alerts with missing fields during rule execution only if configured so for multiple suppress by fields', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:45:00.000Z'; + + const firstExecutionDocuments = [ + // no missing fields + { + host: { name: 'host-a', ip: '127.0.0.11' }, + agent: { name: 'agent-a', version: 10 }, + id, + '@timestamp': timestamp, + }, + { + host: { name: 'host-a', ip: '127.0.0.12' }, + agent: { name: 'agent-a', version: 10 }, + id, + '@timestamp': timestamp, + }, + // missing agent.name + { + host: { name: 'host-a', ip: '127.0.0.21' }, + agent: { version: 10 }, + id, + '@timestamp': timestamp, + }, + { + host: { name: 'host-a', ip: '127.0.0.22' }, + agent: { version: 10 }, + id, + '@timestamp': timestamp, + }, + // missing agent.version + { + host: { name: 'host-a', ip: '127.0.0.31' }, + agent: { name: 'agent-a' }, + id, + '@timestamp': timestamp, + }, + { + host: { name: 'host-a', ip: '127.0.0.32' }, + agent: { name: 'agent-a' }, + id, + '@timestamp': timestamp, + }, + // missing both agent.* + { + host: { name: 'host-a', ip: '127.0.0.41' }, + id, + '@timestamp': timestamp, + }, + { + host: { name: 'host-a', ip: '127.0.0.42' }, + id, + '@timestamp': timestamp, + }, + ]; + + await indexListOfDocuments(firstExecutionDocuments); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: getNonAggRuleQueryWithMetadata(id), + alert_suppression: { + group_by: ['agent.name', 'agent.version'], + missing_fields_strategy: 'doNotSuppress', + }, + from: 'now-35m', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['agent.name', 'agent.version', ALERT_ORIGINAL_TIME], + }); + // from 8 injected, only one should be suppressed + expect(previewAlerts.length).toEqual(7); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: 'agent-a', + }, + { + field: 'agent.version', + value: '10', + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + + // rest of alerts are not suppressed and do not have suppress properties + previewAlerts.slice(1).forEach((previewAlert) => { + const source = previewAlert._source; + expect(source).toHaveProperty('id', id); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_DOCS_COUNT); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_END); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_TERMS); + expect(source).not.toHaveProperty(ALERT_SUPPRESSION_DOCS_COUNT); + }); + }); + + it('should deduplicate alerts while suppressing new ones on rule execution', async () => { + const id = uuidv4(); + const firstTimestamp = '2020-10-28T05:45:00.000Z'; + const secondTimestamp = '2020-10-28T06:10:00.000Z'; + + const firstExecutionDocuments = [ + { + host: { name: 'host-a' }, + id, + '@timestamp': firstTimestamp, + }, + { + host: { name: 'host-a' }, + id, + '@timestamp': firstTimestamp, + }, + ]; + const secondExecutionDocuments = [ + { + host: { name: 'host-a' }, + id, + '@timestamp': secondTimestamp, + }, + { + host: { name: 'host-a' }, + id, + '@timestamp': secondTimestamp, + }, + { + host: { name: 'host-a' }, + id, + '@timestamp': secondTimestamp, + }, + ]; + + await indexListOfDocuments([...firstExecutionDocuments, ...secondExecutionDocuments]); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: getNonAggRuleQueryWithMetadata(id), + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'suppress', + }, + // large look-back time covers all docs + from: 'now-1h', + interval: '30m', + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 2, + }); + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['host.name', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(2); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [ALERT_ORIGINAL_TIME]: firstTimestamp, + [ALERT_SUPPRESSION_START]: firstTimestamp, + [ALERT_SUPPRESSION_END]: firstTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + expect(previewAlerts[1]._source).toEqual({ + ...previewAlerts[1]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'host.name', + value: 'host-a', + }, + ], + [ALERT_ORIGINAL_TIME]: secondTimestamp, + [ALERT_SUPPRESSION_START]: secondTimestamp, + [ALERT_SUPPRESSION_END]: secondTimestamp, + [ALERT_SUPPRESSION_DOCS_COUNT]: 2, + }); + }); + + it('should not suppress more than limited number of alerts (max_signals)', async () => { + const id = uuidv4(); + + await indexGeneratedDocuments({ + docsCount: 12000, + seed: (index) => ({ + id, + '@timestamp': `2020-10-28T06:50:00.${index}Z`, + host: { + name: `host-${index}`, + }, + agent: { name: 'agent-a' }, + }), + }); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: `from ecs_compliant metadata _id, _index, _version ${internalIdPipe( + id + )} | sort @timestamp asc`, + alert_suppression: { + group_by: ['agent.name'], + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + max_signals: 200, + }; + + const { previewId, logs } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T07:00:00.000Z'), + invocationCount: 1, + }); + + expect(logs[0].warnings).toEqual( + expect.arrayContaining([getSuppressionMaxAlertsWarning()]) + ); + + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + sort: ['agent.name', ALERT_ORIGINAL_TIME], + size: 1000, + }); + expect(previewAlerts.length).toEqual(1); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: 'agent-a', + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 200, + }); + }); + + it('should generate up to max_signals alerts', async () => { + const id = uuidv4(); + const timestamp = '2020-10-28T06:05:00.000Z'; + + await indexGeneratedDocuments({ + docsCount: 20000, + seed: (index) => ({ + id, + '@timestamp': timestamp, + host: { + name: `host-${index}`, + }, + 'agent.name': `agent-${index}`, + }), + }); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: getNonAggRuleQueryWithMetadata(id), + alert_suppression: { + group_by: ['agent.name'], + duration: { + value: 300, + unit: 'm', + }, + missing_fields_strategy: 'suppress', + }, + from: 'now-35m', + interval: '30m', + max_signals: 150, + }; + + const { previewId, logs } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + invocationCount: 1, + }); + expect(logs[0].warnings).toEqual( + expect.arrayContaining([getSuppressionMaxAlertsWarning()]) + ); + + const previewAlerts = await getPreviewAlerts({ + es, + previewId, + size: 1000, + sort: ['agent.name', ALERT_ORIGINAL_TIME], + }); + expect(previewAlerts.length).toEqual(150); + }); + }); + + describe('with exceptions', async () => { + afterEach(async () => { + await deleteAllExceptions(supertest, log); + }); + + it('should apply exceptions', async () => { + const id = uuidv4(); + const interval: [string, string] = ['2020-10-28T06:00:00.000Z', '2020-10-28T06:10:00.000Z']; + const doc1 = { agent: { name: 'test-1' }, 'client.ip': '127.0.0.2' }; + const doc2 = { agent: { name: 'test-1' } }; + const doc3 = { agent: { name: 'test-1' }, 'client.ip': '127.0.0.1' }; + + await indexEnhancedDocuments({ documents: [doc1, doc2, doc3], interval, id }); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: `from ecs_compliant ${internalIdPipe(id)} | where agent.name=="test-1"`, + from: 'now-1h', + interval: '1h', + alert_suppression: { + group_by: ['agent.name'], + missing_fields_strategy: 'suppress', + }, + }; + + const { previewId } = await previewRuleWithExceptionEntries({ + supertest, + log, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + entries: [ + [ + { + field: 'client.ip', + operator: 'included', + type: 'match', + value: '127.0.0.1', + }, + ], + ], + }); + + const previewAlerts = await getPreviewAlerts({ es, previewId }); + + expect(previewAlerts.length).toBe(1); + expect(previewAlerts[0]._source).toEqual({ + ...previewAlerts[0]._source, + [ALERT_SUPPRESSION_TERMS]: [ + { + field: 'agent.name', + value: 'test-1', + }, + ], + [ALERT_SUPPRESSION_DOCS_COUNT]: 1, + }); + }); + }); + + describe('alerts enrichment', () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/entity/risks'); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/entity/risks'); + }); + + it('should be enriched with host risk score', async () => { + const id = uuidv4(); + const interval: [string, string] = ['2020-10-28T06:00:00.000Z', '2020-10-28T06:10:00.000Z']; + const doc1 = { host: { name: 'host-0' } }; + + await indexEnhancedDocuments({ documents: [doc1, doc1], interval, id }); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: `from ecs_compliant ${internalIdPipe(id)} | where host.name=="host-0"`, + from: 'now-1h', + interval: '1h', + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'suppress', + }, + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + }); + + const previewAlerts = await getPreviewAlerts({ es, previewId }); + + expect(previewAlerts.length).toBe(1); + + expect(previewAlerts[0]._source).toHaveProperty('host.risk.calculated_level', 'Low'); + expect(previewAlerts[0]._source).toHaveProperty('host.risk.calculated_score_norm', 1); + }); + }); + + describe('with asset criticality', async () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/asset_criticality'); + await kibanaServer.uiSettings.update({ + [ENABLE_ASSET_CRITICALITY_SETTING]: true, + }); + }); + + after(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/asset_criticality'); + }); + + it('should be enriched alert with criticality_level', async () => { + const id = uuidv4(); + const interval: [string, string] = ['2020-10-28T06:00:00.000Z', '2020-10-28T06:10:00.000Z']; + const doc1 = { host: { name: 'host-0' } }; + + await indexEnhancedDocuments({ documents: [doc1, doc1], interval, id }); + + const rule: EsqlRuleCreateProps = { + ...getCreateEsqlRulesSchemaMock('rule-1', true), + query: `from ecs_compliant ${internalIdPipe(id)} | where host.name=="host-0"`, + from: 'now-1h', + interval: '1h', + alert_suppression: { + group_by: ['host.name'], + missing_fields_strategy: 'suppress', + }, + }; + + const { previewId } = await previewRule({ + supertest, + rule, + timeframeEnd: new Date('2020-10-28T06:30:00.000Z'), + }); + + const previewAlerts = await getPreviewAlerts({ es, previewId }); + + expect(previewAlerts.length).toBe(1); + + expect(previewAlerts[0]?._source?.['host.asset.criticality']).toBe('extreme_impact'); + }); + }); + }); +}; diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/index.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/index.ts index 329cef4ac7e8c..3ea2c4e6c9359 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/index.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/index.ts @@ -12,6 +12,7 @@ export default ({ loadTestFile }: FtrProviderContext): void => { loadTestFile(require.resolve('./eql')); loadTestFile(require.resolve('./eql_alert_suppression')); loadTestFile(require.resolve('./esql')); + loadTestFile(require.resolve('./esql_suppression')); loadTestFile(require.resolve('./machine_learning')); loadTestFile(require.resolve('./new_terms')); loadTestFile(require.resolve('./new_terms_alert_suppression')); diff --git a/x-pack/test/security_solution_cypress/config.ts b/x-pack/test/security_solution_cypress/config.ts index 588843b731f45..606df49c4f90e 100644 --- a/x-pack/test/security_solution_cypress/config.ts +++ b/x-pack/test/security_solution_cypress/config.ts @@ -45,6 +45,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { '--xpack.alerting.rules.minimumScheduleInterval.value=1s', '--xpack.ruleRegistry.unsafe.legacyMultiTenancy.enabled=true', `--xpack.securitySolution.enableExperimental=${JSON.stringify([ + 'alertSuppressionForEsqlRuleEnabled', 'bulkCustomHighlightedFieldsEnabled', ])}`, // mock cloud to enable the guided onboarding tour in e2e tests diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/common_flows_suppression_serverless_essentials.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/common_flows_suppression_serverless_essentials.cy.ts index 62a406cd9d466..d6f23687cf418 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/common_flows_suppression_serverless_essentials.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/common_flows_suppression_serverless_essentials.cy.ts @@ -9,6 +9,7 @@ import { selectThresholdRuleType, selectIndicatorMatchType, selectNewTermsRuleType, + selectEsqlRuleType, } from '../../../../tasks/create_new_rule'; import { login } from '../../../../tasks/login'; import { visit } from '../../../../tasks/navigation'; @@ -21,6 +22,7 @@ import { CREATE_RULE_URL } from '../../../../urls/navigation'; describe( 'Detection rules, Alert Suppression for Essentials tier', { + // skipped in MKI as it depends on feature flag alertSuppressionForEsqlRuleEnabled tags: ['@serverless', '@skipInServerlessMKI'], env: { ftrConfig: { @@ -29,6 +31,12 @@ describe( { product_line: 'endpoint', product_tier: 'essentials' }, ], }, + // alertSuppressionForEsqlRuleEnabled feature flag is also enabled in a global config + kbnServerArgs: [ + `--xpack.securitySolution.enableExperimental=${JSON.stringify([ + 'alertSuppressionForEsqlRuleEnabled', + ])}`, + ], }, }, () => { @@ -49,6 +57,9 @@ describe( selectThresholdRuleType(); cy.get(THRESHOLD_ENABLE_SUPPRESSION_CHECKBOX).should('be.enabled'); + + selectEsqlRuleType(); + cy.get(ALERT_SUPPRESSION_FIELDS_INPUT).should('be.enabled'); }); } ); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/common_flows_supression_ess_basic.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/common_flows_supression_ess_basic.cy.ts index fbcc43e4652ae..1f86d6d0dd789 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/common_flows_supression_ess_basic.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/common_flows_supression_ess_basic.cy.ts @@ -14,6 +14,7 @@ import { selectIndicatorMatchType, selectNewTermsRuleType, selectThresholdRuleType, + selectEsqlRuleType, openSuppressionFieldsTooltipAndCheckLicense, } from '../../../../tasks/create_new_rule'; import { startBasicLicense } from '../../../../tasks/api_calls/licensing'; @@ -48,6 +49,9 @@ describe( selectNewTermsRuleType(); openSuppressionFieldsTooltipAndCheckLicense(); + selectEsqlRuleType(); + openSuppressionFieldsTooltipAndCheckLicense(); + selectThresholdRuleType(); cy.get(THRESHOLD_ENABLE_SUPPRESSION_CHECKBOX).should('be.disabled'); cy.get(THRESHOLD_ENABLE_SUPPRESSION_CHECKBOX).parent().trigger('mouseover'); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/esql_rule.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/esql_rule.cy.ts new file mode 100644 index 0000000000000..b8cebae392d38 --- /dev/null +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/esql_rule.cy.ts @@ -0,0 +1,292 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getEsqlRule } from '../../../../objects/rule'; + +import { + RULES_MANAGEMENT_TABLE, + RULE_NAME, + INVESTIGATION_FIELDS_VALUE_ITEM, +} from '../../../../screens/alerts_detection_rules'; +import { + RULE_NAME_HEADER, + RULE_TYPE_DETAILS, + RULE_NAME_OVERRIDE_DETAILS, + DEFINITION_DETAILS, + SUPPRESS_BY_DETAILS, + SUPPRESS_FOR_DETAILS, + SUPPRESS_MISSING_FIELD, +} from '../../../../screens/rule_details'; + +import { ESQL_QUERY_BAR } from '../../../../screens/create_new_rule'; + +import { getDetails, goBackToRulesTable } from '../../../../tasks/rule_details'; +import { expectNumberOfRules } from '../../../../tasks/alerts_detection_rules'; +import { deleteAlertsAndRules } from '../../../../tasks/api_calls/common'; +import { + expandEsqlQueryBar, + fillAboutRuleAndContinue, + fillDefineEsqlRuleAndContinue, + fillScheduleRuleAndContinue, + selectEsqlRuleType, + getDefineContinueButton, + fillEsqlQueryBar, + fillAboutSpecificEsqlRuleAndContinue, + createRuleWithoutEnabling, + expandAdvancedSettings, + fillCustomInvestigationFields, + fillRuleName, + fillDescription, + getAboutContinueButton, + fillAlertSuppressionFields, + selectAlertSuppressionPerInterval, + setAlertSuppressionDuration, + selectDoNotSuppressForMissingFields, + continueFromDefineStep, + fillAboutRuleMinimumAndContinue, + skipScheduleRuleAction, + interceptEsqlQueryFieldsRequest, +} from '../../../../tasks/create_new_rule'; +import { login } from '../../../../tasks/login'; +import { visit } from '../../../../tasks/navigation'; + +import { CREATE_RULE_URL } from '../../../../urls/navigation'; + +// https://github.com/cypress-io/cypress/issues/22113 +// issue is inside monaco editor, used in ES|QL query input +// calling it after visiting page in each tests, seems fixes the issue +// the only other alternative is patching ResizeObserver, which is something I would like to avoid +const workaroundForResizeObserver = () => + cy.on('uncaught:exception', (err) => { + if (err.message.includes('ResizeObserver loop limit exceeded')) { + return false; + } + }); + +describe( + 'Detection ES|QL rules, creation', + { + // skipped in MKI as it depends on feature flag alertSuppressionForEsqlRuleEnabled + // alertSuppressionForEsqlRuleEnabled feature flag is also enabled in a global config + tags: ['@ess', '@serverless', '@skipInServerlessMKI'], + env: { + kbnServerArgs: [ + `--xpack.securitySolution.enableExperimental=${JSON.stringify([ + 'alertSuppressionForEsqlRuleEnabled', + ])}`, + ], + }, + }, + () => { + const rule = getEsqlRule(); + const expectedNumberOfRules = 1; + + describe('creation', () => { + beforeEach(() => { + deleteAlertsAndRules(); + login(); + + visit(CREATE_RULE_URL); + workaroundForResizeObserver(); + }); + + it('creates an ES|QL rule', function () { + selectEsqlRuleType(); + expandEsqlQueryBar(); + + fillDefineEsqlRuleAndContinue(rule); + fillAboutRuleAndContinue(rule); + fillScheduleRuleAndContinue(rule); + createRuleWithoutEnabling(); + + // ensures after rule save ES|QL rule is displayed + cy.get(RULE_NAME_HEADER).should('contain', `${rule.name}`); + getDetails(RULE_TYPE_DETAILS).contains('ES|QL'); + + // ensures newly created rule is displayed in table + goBackToRulesTable(); + + expectNumberOfRules(RULES_MANAGEMENT_TABLE, expectedNumberOfRules); + + cy.get(RULE_NAME).should('have.text', rule.name); + }); + + // this test case is important, since field shown in rule override component are coming from ES|QL query, not data view fields API + it('creates an ES|QL rule and overrides its name', function () { + selectEsqlRuleType(); + expandEsqlQueryBar(); + + fillDefineEsqlRuleAndContinue(rule); + fillAboutSpecificEsqlRuleAndContinue({ ...rule, rule_name_override: 'test_id' }); + fillScheduleRuleAndContinue(rule); + createRuleWithoutEnabling(); + + // ensure rule name override is displayed on details page + getDetails(RULE_NAME_OVERRIDE_DETAILS).should('have.text', 'test_id'); + }); + }); + + describe('ES|QL query validation', () => { + beforeEach(() => { + login(); + visit(CREATE_RULE_URL); + + workaroundForResizeObserver(); + }); + it('shows error when ES|QL query is empty', function () { + selectEsqlRuleType(); + expandEsqlQueryBar(); + getDefineContinueButton().click(); + + cy.get(ESQL_QUERY_BAR).contains('ES|QL query is required'); + }); + + it('proceeds further once invalid query is fixed', function () { + selectEsqlRuleType(); + expandEsqlQueryBar(); + getDefineContinueButton().click(); + + cy.get(ESQL_QUERY_BAR).contains('required'); + + // once correct query typed, we can proceed ot the next step + fillEsqlQueryBar(rule.query); + getDefineContinueButton().click(); + + cy.get(ESQL_QUERY_BAR).should('not.be.visible'); + }); + + it('shows error when non-aggregating ES|QL query does not have metadata operator', function () { + const invalidNonAggregatingQuery = 'from auditbeat* | limit 5'; + selectEsqlRuleType(); + expandEsqlQueryBar(); + fillEsqlQueryBar(invalidNonAggregatingQuery); + getDefineContinueButton().click(); + + cy.get(ESQL_QUERY_BAR).contains( + 'must include the "metadata _id, _version, _index" operator after the source command' + ); + }); + + it('shows error when non-aggregating ES|QL query does not return _id field', function () { + const invalidNonAggregatingQuery = + 'from auditbeat* metadata _id, _version, _index | keep agent.* | limit 5'; + + selectEsqlRuleType(); + expandEsqlQueryBar(); + fillEsqlQueryBar(invalidNonAggregatingQuery); + getDefineContinueButton().click(); + + cy.get(ESQL_QUERY_BAR).contains( + 'must include the "metadata _id, _version, _index" operator after the source command' + ); + }); + + it('shows error when ES|QL query is invalid', function () { + const invalidEsqlQuery = + 'from auditbeat* metadata _id, _version, _index | not_existing_operator'; + visit(CREATE_RULE_URL); + + selectEsqlRuleType(); + expandEsqlQueryBar(); + fillEsqlQueryBar(invalidEsqlQuery); + getDefineContinueButton().click(); + + cy.get(ESQL_QUERY_BAR).contains('Error validating ES|QL'); + }); + }); + + describe('ES|QL investigation fields', () => { + beforeEach(() => { + login(); + visit(CREATE_RULE_URL); + }); + it('shows custom ES|QL field in investigation fields autocomplete and saves it in rule', function () { + const CUSTOM_ESQL_FIELD = '_custom_agent_name'; + const queryWithCustomFields = [ + `from auditbeat* metadata _id, _version, _index`, + `eval ${CUSTOM_ESQL_FIELD} = agent.name`, + `keep _id, _custom_agent_name`, + `limit 5`, + ].join(' | '); + + workaroundForResizeObserver(); + + selectEsqlRuleType(); + expandEsqlQueryBar(); + fillEsqlQueryBar(queryWithCustomFields); + getDefineContinueButton().click(); + + expandAdvancedSettings(); + fillRuleName(); + fillDescription(); + fillCustomInvestigationFields([CUSTOM_ESQL_FIELD]); + getAboutContinueButton().click(); + + fillScheduleRuleAndContinue(rule); + createRuleWithoutEnabling(); + + cy.get(INVESTIGATION_FIELDS_VALUE_ITEM).should('have.text', CUSTOM_ESQL_FIELD); + }); + }); + + describe('Alert suppression', () => { + beforeEach(() => { + login(); + visit(CREATE_RULE_URL); + }); + it('shows custom ES|QL field in investigation fields autocomplete and saves it in rule', function () { + const CUSTOM_ESQL_FIELD = '_custom_agent_name'; + const SUPPRESS_BY_FIELDS = [CUSTOM_ESQL_FIELD, 'agent.type']; + + const queryWithCustomFields = [ + `from auditbeat* metadata _id, _version, _index`, + `eval ${CUSTOM_ESQL_FIELD} = agent.name`, + `drop agent.*`, + ].join(' | '); + + workaroundForResizeObserver(); + + selectEsqlRuleType(); + expandEsqlQueryBar(); + + interceptEsqlQueryFieldsRequest(queryWithCustomFields, 'esqlSuppressionFieldsRequest'); + fillEsqlQueryBar(queryWithCustomFields); + + cy.wait('@esqlSuppressionFieldsRequest'); + fillAlertSuppressionFields(SUPPRESS_BY_FIELDS); + selectAlertSuppressionPerInterval(); + setAlertSuppressionDuration(2, 'h'); + selectDoNotSuppressForMissingFields(); + continueFromDefineStep(); + + // ensures details preview works correctly + cy.get(DEFINITION_DETAILS).within(() => { + getDetails(SUPPRESS_BY_DETAILS).should('have.text', SUPPRESS_BY_FIELDS.join('')); + getDetails(SUPPRESS_FOR_DETAILS).should('have.text', '2h'); + getDetails(SUPPRESS_MISSING_FIELD).should( + 'have.text', + 'Do not suppress alerts for events with missing fields' + ); + }); + + fillAboutRuleMinimumAndContinue(rule); + skipScheduleRuleAction(); + createRuleWithoutEnabling(); + + // ensures rule details displayed correctly after rule created + cy.get(DEFINITION_DETAILS).within(() => { + getDetails(SUPPRESS_BY_DETAILS).should('have.text', SUPPRESS_BY_FIELDS.join('')); + getDetails(SUPPRESS_FOR_DETAILS).should('have.text', '2h'); + getDetails(SUPPRESS_MISSING_FIELD).should( + 'have.text', + 'Do not suppress alerts for events with missing fields' + ); + }); + }); + }); + } +); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/esql_rule_ess.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/esql_rule_ess.cy.ts deleted file mode 100644 index 2e95bb19a0477..0000000000000 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_creation/esql_rule_ess.cy.ts +++ /dev/null @@ -1,219 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { getEsqlRule } from '../../../../objects/rule'; - -import { - RULES_MANAGEMENT_TABLE, - RULE_NAME, - INVESTIGATION_FIELDS_VALUE_ITEM, -} from '../../../../screens/alerts_detection_rules'; -import { - RULE_NAME_HEADER, - RULE_TYPE_DETAILS, - RULE_NAME_OVERRIDE_DETAILS, -} from '../../../../screens/rule_details'; - -import { ESQL_QUERY_BAR } from '../../../../screens/create_new_rule'; - -import { getDetails, goBackToRulesTable } from '../../../../tasks/rule_details'; -import { expectNumberOfRules } from '../../../../tasks/alerts_detection_rules'; -import { deleteAlertsAndRules } from '../../../../tasks/api_calls/common'; -import { - expandEsqlQueryBar, - fillAboutRuleAndContinue, - fillDefineEsqlRuleAndContinue, - fillScheduleRuleAndContinue, - selectEsqlRuleType, - getDefineContinueButton, - fillEsqlQueryBar, - fillAboutSpecificEsqlRuleAndContinue, - createRuleWithoutEnabling, - expandAdvancedSettings, - fillCustomInvestigationFields, - fillRuleName, - fillDescription, - getAboutContinueButton, -} from '../../../../tasks/create_new_rule'; -import { login } from '../../../../tasks/login'; -import { visit } from '../../../../tasks/navigation'; - -import { CREATE_RULE_URL } from '../../../../urls/navigation'; - -// https://github.com/cypress-io/cypress/issues/22113 -// issue is inside monaco editor, used in ES|QL query input -// calling it after visiting page in each tests, seems fixes the issue -// the only other alternative is patching ResizeObserver, which is something I would like to avoid -const workaroundForResizeObserver = () => - cy.on('uncaught:exception', (err) => { - if (err.message.includes('ResizeObserver loop limit exceeded')) { - return false; - } - }); - -describe('Detection ES|QL rules, creation', { tags: ['@ess'] }, () => { - const rule = getEsqlRule(); - const expectedNumberOfRules = 1; - - describe('creation', () => { - beforeEach(() => { - deleteAlertsAndRules(); - login(); - }); - - it('creates an ES|QL rule', function () { - visit(CREATE_RULE_URL); - workaroundForResizeObserver(); - - selectEsqlRuleType(); - expandEsqlQueryBar(); - - fillDefineEsqlRuleAndContinue(rule); - fillAboutRuleAndContinue(rule); - fillScheduleRuleAndContinue(rule); - createRuleWithoutEnabling(); - - // ensures after rule save ES|QL rule is displayed - cy.get(RULE_NAME_HEADER).should('contain', `${rule.name}`); - getDetails(RULE_TYPE_DETAILS).contains('ES|QL'); - - // ensures newly created rule is displayed in table - goBackToRulesTable(); - - expectNumberOfRules(RULES_MANAGEMENT_TABLE, expectedNumberOfRules); - - cy.get(RULE_NAME).should('have.text', rule.name); - }); - - // this test case is important, since field shown in rule override component are coming from ES|QL query, not data view fields API - it('creates an ES|QL rule and overrides its name', function () { - visit(CREATE_RULE_URL); - workaroundForResizeObserver(); - - selectEsqlRuleType(); - expandEsqlQueryBar(); - - fillDefineEsqlRuleAndContinue(rule); - fillAboutSpecificEsqlRuleAndContinue({ ...rule, rule_name_override: 'test_id' }); - fillScheduleRuleAndContinue(rule); - createRuleWithoutEnabling(); - - // ensure rule name override is displayed on details page - getDetails(RULE_NAME_OVERRIDE_DETAILS).should('have.text', 'test_id'); - }); - }); - - describe('ES|QL query validation', () => { - beforeEach(() => { - login(); - visit(CREATE_RULE_URL); - }); - it('shows error when ES|QL query is empty', function () { - workaroundForResizeObserver(); - - selectEsqlRuleType(); - expandEsqlQueryBar(); - getDefineContinueButton().click(); - - cy.get(ESQL_QUERY_BAR).contains('ES|QL query is required'); - }); - - it('proceeds further once invalid query is fixed', function () { - workaroundForResizeObserver(); - - selectEsqlRuleType(); - expandEsqlQueryBar(); - getDefineContinueButton().click(); - - cy.get(ESQL_QUERY_BAR).contains('required'); - - // once correct query typed, we can proceed ot the next step - fillEsqlQueryBar(rule.query); - getDefineContinueButton().click(); - - cy.get(ESQL_QUERY_BAR).should('not.be.visible'); - }); - - it('shows error when non-aggregating ES|QL query does not have metadata operator', function () { - workaroundForResizeObserver(); - - const invalidNonAggregatingQuery = 'from auditbeat* | limit 5'; - selectEsqlRuleType(); - expandEsqlQueryBar(); - fillEsqlQueryBar(invalidNonAggregatingQuery); - getDefineContinueButton().click(); - - cy.get(ESQL_QUERY_BAR).contains( - 'must include the "metadata _id, _version, _index" operator after the source command' - ); - }); - - it('shows error when non-aggregating ES|QL query does not return _id field', function () { - workaroundForResizeObserver(); - - const invalidNonAggregatingQuery = - 'from auditbeat* metadata _id, _version, _index | keep agent.* | limit 5'; - - selectEsqlRuleType(); - expandEsqlQueryBar(); - fillEsqlQueryBar(invalidNonAggregatingQuery); - getDefineContinueButton().click(); - - cy.get(ESQL_QUERY_BAR).contains( - 'must include the "metadata _id, _version, _index" operator after the source command' - ); - }); - - it('shows error when ES|QL query is invalid', function () { - workaroundForResizeObserver(); - const invalidEsqlQuery = - 'from auditbeat* metadata _id, _version, _index | not_existing_operator'; - visit(CREATE_RULE_URL); - - selectEsqlRuleType(); - expandEsqlQueryBar(); - fillEsqlQueryBar(invalidEsqlQuery); - getDefineContinueButton().click(); - - cy.get(ESQL_QUERY_BAR).contains('Error validating ES|QL'); - }); - }); - - describe('ES|QL investigation fields', () => { - beforeEach(() => { - login(); - visit(CREATE_RULE_URL); - }); - it('shows custom ES|QL field in investigation fields autocomplete and saves it in rule', function () { - const CUSTOM_ESQL_FIELD = '_custom_agent_name'; - const queryWithCustomFields = [ - `from auditbeat* metadata _id, _version, _index`, - `eval ${CUSTOM_ESQL_FIELD} = agent.name`, - `keep _id, _custom_agent_name`, - `limit 5`, - ].join(' | '); - - workaroundForResizeObserver(); - - selectEsqlRuleType(); - expandEsqlQueryBar(); - fillEsqlQueryBar(queryWithCustomFields); - getDefineContinueButton().click(); - - expandAdvancedSettings(); - fillRuleName(); - fillDescription(); - fillCustomInvestigationFields([CUSTOM_ESQL_FIELD]); - getAboutContinueButton().click(); - - fillScheduleRuleAndContinue(rule); - createRuleWithoutEnabling(); - - cy.get(INVESTIGATION_FIELDS_VALUE_ITEM).should('have.text', CUSTOM_ESQL_FIELD); - }); - }); -}); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/esql_rule.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/esql_rule.cy.ts index 265263ba495c6..a255ce289b1a7 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/esql_rule.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine/rule_edit/esql_rule.cy.ts @@ -7,9 +7,22 @@ import { getEsqlRule } from '../../../../objects/rule'; -import { ESQL_QUERY_DETAILS, RULE_NAME_OVERRIDE_DETAILS } from '../../../../screens/rule_details'; +import { + ESQL_QUERY_DETAILS, + RULE_NAME_OVERRIDE_DETAILS, + SUPPRESS_FOR_DETAILS, + DEFINITION_DETAILS, + SUPPRESS_MISSING_FIELD, + SUPPRESS_BY_DETAILS, + DETAILS_TITLE, +} from '../../../../screens/rule_details'; -import { ESQL_QUERY_BAR } from '../../../../screens/create_new_rule'; +import { + ESQL_QUERY_BAR, + ALERT_SUPPRESSION_DURATION_INPUT, + ALERT_SUPPRESSION_FIELDS, + ALERT_SUPPRESSION_MISSING_FIELDS_SUPPRESS, +} from '../../../../screens/create_new_rule'; import { createRule } from '../../../../tasks/api_calls/rules'; @@ -17,12 +30,15 @@ import { RULES_MANAGEMENT_URL } from '../../../../urls/rules_management'; import { getDetails } from '../../../../tasks/rule_details'; import { deleteAlertsAndRules } from '../../../../tasks/api_calls/common'; import { - clearEsqlQueryBar, expandEsqlQueryBar, fillEsqlQueryBar, fillOverrideEsqlRuleName, goToAboutStepTab, expandAdvancedSettings, + selectAlertSuppressionPerRuleExecution, + selectDoNotSuppressForMissingFields, + fillAlertSuppressionFields, + interceptEsqlQueryFieldsRequest, } from '../../../../tasks/create_new_rule'; import { login } from '../../../../tasks/login'; @@ -33,63 +49,160 @@ import { visit } from '../../../../tasks/navigation'; const rule = getEsqlRule(); -const expectedValidEsqlQuery = 'from auditbeat* | stats count(event.category) by event.category'; - -describe('Detection ES|QL rules, edit', { tags: ['@ess'] }, () => { - beforeEach(() => { - login(); - deleteAlertsAndRules(); - createRule(rule); - }); - - it('edits ES|QL rule and checks details page', () => { - visit(RULES_MANAGEMENT_URL); - editFirstRule(); - expandEsqlQueryBar(); - // ensure once edit form opened, correct query is displayed in ES|QL input - cy.get(ESQL_QUERY_BAR).contains(rule.query); - - clearEsqlQueryBar(); - fillEsqlQueryBar(expectedValidEsqlQuery); - - saveEditedRule(); - - // ensure updated query is displayed on details page - getDetails(ESQL_QUERY_DETAILS).should('have.text', expectedValidEsqlQuery); - }); - - it('edits ES|QL rule query and override rule name with new property', () => { - visit(RULES_MANAGEMENT_URL); - editFirstRule(); - clearEsqlQueryBar(); - fillEsqlQueryBar(expectedValidEsqlQuery); - - goToAboutStepTab(); - expandAdvancedSettings(); - fillOverrideEsqlRuleName('event.category'); - - saveEditedRule(); - - // ensure rule name override is displayed on details page - getDetails(RULE_NAME_OVERRIDE_DETAILS).should('have.text', 'event.category'); - }); - - it('adds ES|QL override rule name on edit', () => { - visit(RULES_MANAGEMENT_URL); - editFirstRule(); - - expandEsqlQueryBar(); - // ensure once edit form opened, correct query is displayed in ES|QL input - cy.get(ESQL_QUERY_BAR).contains(rule.query); - - goToAboutStepTab(); - expandAdvancedSettings(); - // this field defined to be returned in rule query - fillOverrideEsqlRuleName('test_id'); - - saveEditedRule(); - - // ensure rule name override is displayed on details page - getDetails(RULE_NAME_OVERRIDE_DETAILS).should('have.text', 'test_id'); - }); -}); +const expectedValidEsqlQuery = + 'from auditbeat* | stats _count=count(event.category) by event.category'; + +// skipped in MKI as it depends on feature flag alertSuppressionForEsqlRuleEnabled +// alertSuppressionForEsqlRuleEnabled feature flag is also enabled in a global config +describe( + 'Detection ES|QL rules, edit', + { + tags: ['@ess', '@serverless', '@skipInServerlessMKI'], + env: { + kbnServerArgs: [ + `--xpack.securitySolution.enableExperimental=${JSON.stringify([ + 'alertSuppressionForEsqlRuleEnabled', + ])}`, + ], + }, + }, + () => { + beforeEach(() => { + login(); + deleteAlertsAndRules(); + createRule(rule); + + visit(RULES_MANAGEMENT_URL); + editFirstRule(); + }); + + it('edits ES|QL rule and checks details page', () => { + expandEsqlQueryBar(); + // ensure once edit form opened, correct query is displayed in ES|QL input + cy.get(ESQL_QUERY_BAR).contains(rule.query); + + fillEsqlQueryBar(expectedValidEsqlQuery); + + saveEditedRule(); + + // ensure updated query is displayed on details page + getDetails(ESQL_QUERY_DETAILS).should('have.text', expectedValidEsqlQuery); + }); + + it('edits ES|QL rule query and override rule name with new property', () => { + fillEsqlQueryBar(expectedValidEsqlQuery); + + goToAboutStepTab(); + expandAdvancedSettings(); + fillOverrideEsqlRuleName('event.category'); + + saveEditedRule(); + + // ensure rule name override is displayed on details page + getDetails(RULE_NAME_OVERRIDE_DETAILS).should('have.text', 'event.category'); + }); + + it('adds ES|QL override rule name on edit', () => { + expandEsqlQueryBar(); + // ensure once edit form opened, correct query is displayed in ES|QL input + cy.get(ESQL_QUERY_BAR).contains(rule.query); + + goToAboutStepTab(); + expandAdvancedSettings(); + // this field defined to be returned in rule query + fillOverrideEsqlRuleName('test_id'); + + saveEditedRule(); + + // ensure rule name override is displayed on details page + getDetails(RULE_NAME_OVERRIDE_DETAILS).should('have.text', 'test_id'); + }); + + describe('with configured suppression', () => { + const SUPPRESS_BY_FIELDS = ['event.category']; + const NEW_SUPPRESS_BY_FIELDS = ['event.category', '_count']; + + beforeEach(() => { + deleteAlertsAndRules(); + createRule({ + ...rule, + query: expectedValidEsqlQuery, + alert_suppression: { + group_by: SUPPRESS_BY_FIELDS, + duration: { value: 3, unit: 'h' }, + missing_fields_strategy: 'suppress', + }, + }); + }); + + it('displays suppress options correctly on edit form and allows its editing', () => { + visit(RULES_MANAGEMENT_URL); + + interceptEsqlQueryFieldsRequest(expectedValidEsqlQuery, 'esqlSuppressionFieldsRequest'); + editFirstRule(); + + // check saved suppression settings + cy.get(ALERT_SUPPRESSION_DURATION_INPUT).eq(0).should('be.enabled').should('have.value', 3); + cy.get(ALERT_SUPPRESSION_DURATION_INPUT) + .eq(1) + .should('be.enabled') + .should('have.value', 'h'); + cy.get(ALERT_SUPPRESSION_FIELDS).should('contain', SUPPRESS_BY_FIELDS.join('')); + cy.get(ALERT_SUPPRESSION_MISSING_FIELDS_SUPPRESS).should('be.checked'); + + selectAlertSuppressionPerRuleExecution(); + selectDoNotSuppressForMissingFields(); + + cy.wait('@esqlSuppressionFieldsRequest'); + fillAlertSuppressionFields(['_count']); + + saveEditedRule(); + + cy.get(DEFINITION_DETAILS).within(() => { + getDetails(SUPPRESS_BY_DETAILS).should('have.text', NEW_SUPPRESS_BY_FIELDS.join('')); + getDetails(SUPPRESS_FOR_DETAILS).should('have.text', 'One rule execution'); + getDetails(SUPPRESS_MISSING_FIELD).should( + 'have.text', + 'Do not suppress alerts for events with missing fields' + ); + }); + }); + }); + + describe('without suppression', () => { + const SUPPRESS_BY_FIELDS = ['event.category']; + + beforeEach(() => { + deleteAlertsAndRules(); + createRule({ + ...rule, + query: expectedValidEsqlQuery, + }); + }); + + it('enables suppression on time interval', () => { + visit(RULES_MANAGEMENT_URL); + + interceptEsqlQueryFieldsRequest(expectedValidEsqlQuery, 'esqlSuppressionFieldsRequest'); + editFirstRule(); + + cy.wait('@esqlSuppressionFieldsRequest'); + fillAlertSuppressionFields(SUPPRESS_BY_FIELDS); + + saveEditedRule(); + + cy.get(DEFINITION_DETAILS).within(() => { + getDetails(SUPPRESS_BY_DETAILS).should('have.text', SUPPRESS_BY_FIELDS.join('')); + getDetails(SUPPRESS_FOR_DETAILS).should('have.text', 'One rule execution'); + getDetails(SUPPRESS_MISSING_FIELD).should( + 'have.text', + 'Suppress and group alerts for events with missing fields' + ); + + // suppression functionality should be under Tech Preview + cy.contains(DETAILS_TITLE, SUPPRESS_FOR_DETAILS).contains('Technical Preview'); + }); + }); + }); + } +); diff --git a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/prebuilt_rules_preview.cy.ts b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/prebuilt_rules_preview.cy.ts index 15229445e54f0..13af60594d103 100644 --- a/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/prebuilt_rules_preview.cy.ts +++ b/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/rule_management/prebuilt_rules/prebuilt_rules_preview.cy.ts @@ -337,6 +337,14 @@ describe('Detection rules, Prebuilt Rules Installation and Update workflow', () type: 'esql', language: 'esql', query: 'FROM .alerts-security.alerts-default | STATS count = COUNT(@timestamp) BY @timestamp', + alert_suppression: { + group_by: [ + 'Endpoint.policy.applied.artifacts.global.identifiers.name', + 'Endpoint.policy.applied.id', + ], + duration: { unit: 'm', value: 5 }, + missing_fields_strategy: 'suppress', + }, }); const RULE_WITHOUT_INVESTIGATION_AND_SETUP_GUIDES = createRuleAssetSavedObject({ @@ -621,25 +629,23 @@ describe('Detection rules, Prebuilt Rules Installation and Update workflow', () const { query } = NEW_TERMS_INDEX_PATTERN_RULE['security-rule'] as { query: string }; assertCustomQueryPropertyShown(query); }); - }); - describe( - 'Skip in Serverless environment', - { tags: TEST_ENV_TAGS.filter((tag) => tag !== '@serverless') }, - () => { - /* Serverless environment doesn't support ESQL rules just yet */ - it('ESQL rule properties', () => { - clickAddElasticRulesButton(); + it('ESQL rule properties', () => { + clickAddElasticRulesButton(); - openRuleInstallPreview(ESQL_RULE['security-rule'].name); + openRuleInstallPreview(ESQL_RULE['security-rule'].name); - assertCommonPropertiesShown(commonProperties); + assertCommonPropertiesShown(commonProperties); - const { query } = ESQL_RULE['security-rule'] as { query: string }; - assertEsqlQueryPropertyShown(query); - }); - } - ); + const { query } = ESQL_RULE['security-rule'] as { query: string }; + assertEsqlQueryPropertyShown(query); + + const { alert_suppression: alertSuppression } = ESQL_RULE['security-rule'] as { + alert_suppression: AlertSuppression; + }; + assertAlertSuppressionPropertiesShown(alertSuppression); + }); + }); }); }); @@ -1049,26 +1055,24 @@ describe('Detection rules, Prebuilt Rules Installation and Update workflow', () }; assertCustomQueryPropertyShown(query); }); - }); - describe( - 'Skip in Serverless environment', - { tags: TEST_ENV_TAGS.filter((tag) => tag !== '@serverless') }, - () => { - /* Serverless environment doesn't support ESQL rules just yet */ - it('ESQL rule properties', () => { - clickRuleUpdatesTab(); + it('ESQL rule properties', () => { + clickRuleUpdatesTab(); + + openRuleUpdatePreview(UPDATED_ESQL_RULE['security-rule'].name); + selectPreviewTab(PREVIEW_TABS.OVERVIEW); - openRuleUpdatePreview(UPDATED_ESQL_RULE['security-rule'].name); - selectPreviewTab(PREVIEW_TABS.OVERVIEW); + assertCommonPropertiesShown(commonProperties); - assertCommonPropertiesShown(commonProperties); + const { query } = UPDATED_ESQL_RULE['security-rule'] as { query: string }; + assertEsqlQueryPropertyShown(query); - const { query } = UPDATED_ESQL_RULE['security-rule'] as { query: string }; - assertEsqlQueryPropertyShown(query); - }); - } - ); + const { alert_suppression: alertSuppression } = UPDATED_ESQL_RULE['security-rule'] as { + alert_suppression: AlertSuppression; + }; + assertAlertSuppressionPropertiesShown(alertSuppression); + }); + }); }); describe('Viewing rule changes in JSON diff view', { tags: TEST_ENV_TAGS }, () => { diff --git a/x-pack/test/security_solution_cypress/cypress/objects/rule.ts b/x-pack/test/security_solution_cypress/cypress/objects/rule.ts index 0b9a4ddedf04d..fcbaaca3d1dde 100644 --- a/x-pack/test/security_solution_cypress/cypress/objects/rule.ts +++ b/x-pack/test/security_solution_cypress/cypress/objects/rule.ts @@ -406,7 +406,7 @@ export const getEsqlRule = ( ): EsqlRuleCreateProps => ({ type: 'esql', language: 'esql', - query: 'from auditbeat-* [metadata _id, _version, _index] | keep agent.*,_id | eval test_id=_id', + query: 'from auditbeat-* metadata _id, _version, _index | keep agent.*,_id | eval test_id=_id', name: 'ES|QL Rule', description: 'The new rule description.', severity: 'high', diff --git a/x-pack/test/security_solution_cypress/cypress/tasks/create_new_rule.ts b/x-pack/test/security_solution_cypress/cypress/tasks/create_new_rule.ts index a71d990ec31a9..181f5dfa22eb1 100644 --- a/x-pack/test/security_solution_cypress/cypress/tasks/create_new_rule.ts +++ b/x-pack/test/security_solution_cypress/cypress/tasks/create_new_rule.ts @@ -613,16 +613,23 @@ export const fillDefineNewTermsRuleAndContinue = (rule: NewTermsRuleCreateProps) cy.get(DEFINE_CONTINUE_BUTTON).should('exist').click({ force: true }); }; -export const fillEsqlQueryBar = (query: string) => { +const typeEsqlQueryBar = (query: string) => { // eslint-disable-next-line cypress/no-force cy.get(ESQL_QUERY_BAR_INPUT_AREA).should('not.be.disabled').type(query, { force: true }); }; -export const clearEsqlQueryBar = () => { - // monaco editor under the hood is quite complex in matter to clear it - // underlying textarea holds just the last character of query displayed in search bar - // in order to clear it - it requires to select all text within editor and type in it - fillEsqlQueryBar(Cypress.platform === 'darwin' ? '{cmd}a' : '{ctrl}a'); +/** + * clears ES|QL search bar first + * types new query + */ +export const fillEsqlQueryBar = (query: string) => { + // before typing anything in query bar, we need to clear it + // Since first click on ES|QL query bar trigger re-render. We need to clear search bar during second attempt + typeEsqlQueryBar(' '); + typeEsqlQueryBar(Cypress.platform === 'darwin' ? '{cmd}a{del}' : '{ctrl}a{del}'); + + // only after this query can be safely typed + typeEsqlQueryBar(query); }; /** @@ -939,6 +946,20 @@ export const openSuppressionFieldsTooltipAndCheckLicense = () => { cy.get(TOOLTIP).contains('Platinum license'); }; +/** + * intercepts /internal/bsearch request that contains esqlQuery and adds alias to it + */ +export const interceptEsqlQueryFieldsRequest = ( + esqlQuery: string, + alias: string = 'esqlQueryFields' +) => { + cy.intercept('POST', '/internal/bsearch?*', (req) => { + if (req.body?.batch?.[0]?.request?.params?.query?.includes?.(esqlQuery)) { + req.alias = alias; + } + }); +}; + export const checkLoadQueryDynamically = () => { cy.get(LOAD_QUERY_DYNAMICALLY_CHECKBOX).click({ force: true }); cy.get(LOAD_QUERY_DYNAMICALLY_CHECKBOX).should('be.checked'); diff --git a/x-pack/test/security_solution_cypress/serverless_config.ts b/x-pack/test/security_solution_cypress/serverless_config.ts index 51462be717fc8..04a15e49d070a 100644 --- a/x-pack/test/security_solution_cypress/serverless_config.ts +++ b/x-pack/test/security_solution_cypress/serverless_config.ts @@ -35,6 +35,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { { product_line: 'cloud', product_tier: 'complete' }, ])}`, `--xpack.securitySolution.enableExperimental=${JSON.stringify([ + 'alertSuppressionForEsqlRuleEnabled', 'bulkCustomHighlightedFieldsEnabled', ])}`, ],