diff --git a/x-pack/plugins/osquery/common/ecs/rule/index.ts b/x-pack/plugins/osquery/common/ecs/rule/index.ts index 3a764a836e9b3..51d7722a3ecde 100644 --- a/x-pack/plugins/osquery/common/ecs/rule/index.ts +++ b/x-pack/plugins/osquery/common/ecs/rule/index.ts @@ -28,7 +28,7 @@ export interface RuleEcs { tags?: string[]; threat?: unknown; threshold?: { - field: string; + field: string | string[]; value: number; }; type?: string[]; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts index aade8be4f503f..d97820f010a80 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts @@ -459,12 +459,21 @@ export type Threats = t.TypeOf; export const threatsOrUndefined = t.union([threats, t.undefined]); export type ThreatsOrUndefined = t.TypeOf; -export const threshold = t.exact( - t.type({ - field: t.string, - value: PositiveIntegerGreaterThanZero, - }) -); +export const threshold = t.intersection([ + t.exact( + t.type({ + field: t.union([t.string, t.array(t.string)]), + value: PositiveIntegerGreaterThanZero, + }) + ), + t.exact( + t.partial({ + cardinality_field: t.union([t.string, t.array(t.string), t.undefined, t.null]), + cardinality_value: t.union([PositiveInteger, t.undefined, t.null]), // TODO: cardinality_value should be set if cardinality_field is set + }) + ), +]); +// TODO: codec to transform threshold field string to string[] ? export type Threshold = t.TypeOf; export const thresholdOrUndefined = t.union([threshold, t.undefined]); diff --git a/x-pack/plugins/security_solution/common/ecs/rule/index.ts b/x-pack/plugins/security_solution/common/ecs/rule/index.ts index 3a764a836e9b3..5463b21f6b7f7 100644 --- a/x-pack/plugins/security_solution/common/ecs/rule/index.ts +++ b/x-pack/plugins/security_solution/common/ecs/rule/index.ts @@ -27,10 +27,7 @@ export interface RuleEcs { severity?: string[]; tags?: string[]; threat?: unknown; - threshold?: { - field: string; - value: number; - }; + threshold?: unknown; type?: string[]; size?: string[]; to?: string[]; diff --git a/x-pack/plugins/security_solution/common/ecs/signal/index.ts b/x-pack/plugins/security_solution/common/ecs/signal/index.ts index eb5e629a1abcf..45e1f04d2b405 100644 --- a/x-pack/plugins/security_solution/common/ecs/signal/index.ts +++ b/x-pack/plugins/security_solution/common/ecs/signal/index.ts @@ -14,4 +14,5 @@ export interface SignalEcs { group?: { id?: string[]; }; + threshold_result?: unknown; } diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram/index.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram/index.ts index df6543ddbce90..c71108d58d980 100644 --- a/x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram/index.ts +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram/index.ts @@ -36,7 +36,14 @@ export interface MatrixHistogramRequestOptions extends RequestBasicOptions { timerange: TimerangeInput; histogramType: MatrixHistogramType; stackByField: string; - threshold?: { field: string | undefined; value: number } | undefined; + threshold?: + | { + field: string | string[] | undefined; + value: number; + cardinality_field?: string | undefined; + cardinality_value?: number | undefined; + } + | undefined; inspect?: Maybe; isPtrIncluded?: boolean; } diff --git a/x-pack/plugins/security_solution/cypress/integration/detection_rules/indicator_match_rule.spec.ts b/x-pack/plugins/security_solution/cypress/integration/detection_rules/indicator_match_rule.spec.ts index a69f808001800..bc52be678347a 100644 --- a/x-pack/plugins/security_solution/cypress/integration/detection_rules/indicator_match_rule.spec.ts +++ b/x-pack/plugins/security_solution/cypress/integration/detection_rules/indicator_match_rule.spec.ts @@ -98,359 +98,375 @@ import { loginAndWaitForPageWithoutDateRange } from '../../tasks/login'; import { DETECTIONS_URL, RULE_CREATION } from '../../urls/navigation'; -describe('Detection rules, Indicator Match', () => { - const expectedUrls = newThreatIndicatorRule.referenceUrls.join(''); - const expectedFalsePositives = newThreatIndicatorRule.falsePositivesExamples.join(''); - const expectedTags = newThreatIndicatorRule.tags.join(''); - const expectedMitre = formatMitreAttackDescription(newThreatIndicatorRule.mitre); - const expectedNumberOfRules = 1; - const expectedNumberOfAlerts = 1; - - before(() => { - cleanKibana(); - esArchiverLoad('threat_indicator'); - esArchiverLoad('threat_data'); - }); - after(() => { - esArchiverUnload('threat_indicator'); - esArchiverUnload('threat_data'); - }); - - describe('Creating new indicator match rules', () => { - beforeEach(() => { - loginAndWaitForPageWithoutDateRange(RULE_CREATION); - selectIndicatorMatchType(); +// Skipped for 7.12 FF - flaky tests +describe.skip('indicator match', () => { + describe('Detection rules, Indicator Match', () => { + const expectedUrls = newThreatIndicatorRule.referenceUrls.join(''); + const expectedFalsePositives = newThreatIndicatorRule.falsePositivesExamples.join(''); + const expectedTags = newThreatIndicatorRule.tags.join(''); + const expectedMitre = formatMitreAttackDescription(newThreatIndicatorRule.mitre); + const expectedNumberOfRules = 1; + const expectedNumberOfAlerts = 1; + + before(() => { + cleanKibana(); + esArchiverLoad('threat_indicator'); + esArchiverLoad('threat_data'); }); - - describe('Index patterns', () => { - it('Contains a predefined index pattern', () => { - getIndicatorIndex().should('have.text', indexPatterns.join('')); - }); - - it('Does NOT show invalidation text on initial page load if indicator index pattern is filled out', () => { - getIndicatorIndicatorIndex().type(`${newThreatIndicatorRule.indicatorIndexPattern}{enter}`); - getDefineContinueButton().click(); - getIndexPatternInvalidationText().should('not.exist'); - }); - - it('Shows invalidation text when you try to continue without filling it out', () => { - getIndexPatternClearButton().click(); - getIndicatorIndicatorIndex().type(`${newThreatIndicatorRule.indicatorIndexPattern}{enter}`); - getDefineContinueButton().click(); - getIndexPatternInvalidationText().should('exist'); - }); + after(() => { + esArchiverUnload('threat_indicator'); + esArchiverUnload('threat_data'); }); - describe('Indicator index patterns', () => { - it('Contains empty index pattern', () => { - getIndicatorIndicatorIndex().should('have.text', ''); - }); - - it('Does NOT show invalidation text on initial page load', () => { - getIndexPatternInvalidationText().should('not.exist'); - }); - - it('Shows invalidation text if you try to continue without filling it out', () => { - getDefineContinueButton().click(); - getIndexPatternInvalidationText().should('exist'); + describe('Creating new indicator match rules', () => { + beforeEach(() => { + loginAndWaitForPageWithoutDateRange(RULE_CREATION); + selectIndicatorMatchType(); }); - }); - describe('custom query input', () => { - it('Has a default set of *:*', () => { - getCustomQueryInput().should('have.text', '*:*'); - }); + describe('Index patterns', () => { + it('Contains a predefined index pattern', () => { + getIndicatorIndex().should('have.text', indexPatterns.join('')); + }); - it('Shows invalidation text if text is removed', () => { - getCustomQueryInput().type('{selectall}{del}'); - getCustomQueryInvalidationText().should('exist'); - }); - }); + it('Does NOT show invalidation text on initial page load if indicator index pattern is filled out', () => { + getIndicatorIndicatorIndex().type( + `${newThreatIndicatorRule.indicatorIndexPattern}{enter}` + ); + getDefineContinueButton().click(); + getIndexPatternInvalidationText().should('not.exist'); + }); - describe('custom indicator query input', () => { - it('Has a default set of *:*', () => { - getCustomIndicatorQueryInput().should('have.text', '*:*'); + it('Shows invalidation text when you try to continue without filling it out', () => { + getIndexPatternClearButton().click(); + getIndicatorIndicatorIndex().type( + `${newThreatIndicatorRule.indicatorIndexPattern}{enter}` + ); + getDefineContinueButton().click(); + getIndexPatternInvalidationText().should('exist'); + }); }); - it('Shows invalidation text if text is removed', () => { - getCustomIndicatorQueryInput().type('{selectall}{del}'); - getCustomQueryInvalidationText().should('exist'); - }); - }); + describe('Indicator index patterns', () => { + it('Contains empty index pattern', () => { + getIndicatorIndicatorIndex().should('have.text', ''); + }); - describe('Indicator mapping', () => { - beforeEach(() => { - fillIndexAndIndicatorIndexPattern( - newThreatIndicatorRule.index, - newThreatIndicatorRule.indicatorIndexPattern - ); - }); + it('Does NOT show invalidation text on initial page load', () => { + getIndexPatternInvalidationText().should('not.exist'); + }); - it('Does NOT show invalidation text on initial page load', () => { - getIndicatorInvalidationText().should('not.exist'); + it('Shows invalidation text if you try to continue without filling it out', () => { + getDefineContinueButton().click(); + getIndexPatternInvalidationText().should('exist'); + }); }); - it('Shows invalidation text when you try to press continue without filling anything out', () => { - getDefineContinueButton().click(); - getIndicatorAtLeastOneInvalidationText().should('exist'); - }); + describe('custom query input', () => { + it('Has a default set of *:*', () => { + getCustomQueryInput().should('have.text', '*:*'); + }); - it('Shows invalidation text when the "AND" button is pressed and both the mappings are blank', () => { - getIndicatorAndButton().click(); - getIndicatorInvalidationText().should('exist'); + it('Shows invalidation text if text is removed', () => { + getCustomQueryInput().type('{selectall}{del}'); + getCustomQueryInvalidationText().should('exist'); + }); }); - it('Shows invalidation text when the "OR" button is pressed and both the mappings are blank', () => { - getIndicatorOrButton().click(); - getIndicatorInvalidationText().should('exist'); - }); + describe('custom indicator query input', () => { + it('Has a default set of *:*', () => { + getCustomIndicatorQueryInput().should('have.text', '*:*'); + }); - it('Does NOT show invalidation text when there is a valid "index field" and a valid "indicator index field"', () => { - fillIndicatorMatchRow({ - indexField: newThreatIndicatorRule.indicatorMapping, - indicatorIndexField: newThreatIndicatorRule.indicatorIndexField, + it('Shows invalidation text if text is removed', () => { + getCustomIndicatorQueryInput().type('{selectall}{del}'); + getCustomQueryInvalidationText().should('exist'); }); - getDefineContinueButton().click(); - getIndicatorInvalidationText().should('not.exist'); }); - it('Shows invalidation text when there is an invalid "index field" and a valid "indicator index field"', () => { - fillIndicatorMatchRow({ - indexField: 'non-existent-value', - indicatorIndexField: newThreatIndicatorRule.indicatorIndexField, - validColumns: 'indicatorField', + describe('Indicator mapping', () => { + beforeEach(() => { + fillIndexAndIndicatorIndexPattern( + newThreatIndicatorRule.index, + newThreatIndicatorRule.indicatorIndexPattern + ); }); - getDefineContinueButton().click(); - getIndicatorInvalidationText().should('exist'); - }); - it('Shows invalidation text when there is a valid "index field" and an invalid "indicator index field"', () => { - fillIndicatorMatchRow({ - indexField: newThreatIndicatorRule.indicatorMapping, - indicatorIndexField: 'non-existent-value', - validColumns: 'indexField', + it('Does NOT show invalidation text on initial page load', () => { + getIndicatorInvalidationText().should('not.exist'); }); - getDefineContinueButton().click(); - getIndicatorInvalidationText().should('exist'); - }); - it('Deletes the first row when you have two rows. Both rows valid rows of "index fields" and valid "indicator index fields". The second row should become the first row', () => { - fillIndicatorMatchRow({ - indexField: newThreatIndicatorRule.indicatorMapping, - indicatorIndexField: newThreatIndicatorRule.indicatorIndexField, + it('Shows invalidation text when you try to press continue without filling anything out', () => { + getDefineContinueButton().click(); + getIndicatorAtLeastOneInvalidationText().should('exist'); }); - getIndicatorAndButton().click(); - fillIndicatorMatchRow({ - rowNumber: 2, - indexField: 'agent.name', - indicatorIndexField: newThreatIndicatorRule.indicatorIndexField, - validColumns: 'indicatorField', + + it('Shows invalidation text when the "AND" button is pressed and both the mappings are blank', () => { + getIndicatorAndButton().click(); + getIndicatorInvalidationText().should('exist'); }); - getIndicatorDeleteButton().click(); - getIndicatorIndexComboField().should('have.text', 'agent.name'); - getIndicatorMappingComboField().should( - 'have.text', - newThreatIndicatorRule.indicatorIndexField - ); - getIndicatorIndexComboField(2).should('not.exist'); - getIndicatorMappingComboField(2).should('not.exist'); - }); - it('Deletes the first row when you have two rows. Both rows have valid "index fields" and invalid "indicator index fields". The second row should become the first row', () => { - fillIndicatorMatchRow({ - indexField: newThreatIndicatorRule.indicatorMapping, - indicatorIndexField: 'non-existent-value', - validColumns: 'indexField', + it('Shows invalidation text when the "OR" button is pressed and both the mappings are blank', () => { + getIndicatorOrButton().click(); + getIndicatorInvalidationText().should('exist'); }); - getIndicatorAndButton().click(); - fillIndicatorMatchRow({ - rowNumber: 2, - indexField: newThreatIndicatorRule.indicatorMapping, - indicatorIndexField: 'second-non-existent-value', - validColumns: 'indexField', + + it('Does NOT show invalidation text when there is a valid "index field" and a valid "indicator index field"', () => { + fillIndicatorMatchRow({ + indexField: newThreatIndicatorRule.indicatorMapping, + indicatorIndexField: newThreatIndicatorRule.indicatorIndexField, + }); + getDefineContinueButton().click(); + getIndicatorInvalidationText().should('not.exist'); }); - getIndicatorDeleteButton().click(); - getIndicatorMappingComboField().should('have.text', 'second-non-existent-value'); - getIndicatorIndexComboField(2).should('not.exist'); - getIndicatorMappingComboField(2).should('not.exist'); - }); - it('Deletes the first row when you have two rows. Both rows have valid "indicator index fields" and invalid "index fields". The second row should become the first row', () => { - fillIndicatorMatchRow({ - indexField: 'non-existent-value', - indicatorIndexField: newThreatIndicatorRule.indicatorIndexField, - validColumns: 'indicatorField', + it('Shows invalidation text when there is an invalid "index field" and a valid "indicator index field"', () => { + fillIndicatorMatchRow({ + indexField: 'non-existent-value', + indicatorIndexField: newThreatIndicatorRule.indicatorIndexField, + validColumns: 'indicatorField', + }); + getDefineContinueButton().click(); + getIndicatorInvalidationText().should('exist'); }); - getIndicatorAndButton().click(); - fillIndicatorMatchRow({ - rowNumber: 2, - indexField: 'second-non-existent-value', - indicatorIndexField: newThreatIndicatorRule.indicatorIndexField, - validColumns: 'indicatorField', + + it('Shows invalidation text when there is a valid "index field" and an invalid "indicator index field"', () => { + fillIndicatorMatchRow({ + indexField: newThreatIndicatorRule.indicatorMapping, + indicatorIndexField: 'non-existent-value', + validColumns: 'indexField', + }); + getDefineContinueButton().click(); + getIndicatorInvalidationText().should('exist'); }); - getIndicatorDeleteButton().click(); - getIndicatorIndexComboField().should('have.text', 'second-non-existent-value'); - getIndicatorIndexComboField(2).should('not.exist'); - getIndicatorMappingComboField(2).should('not.exist'); - }); - it('Deletes the first row of data but not the UI elements and the text defaults back to the placeholder of Search', () => { - fillIndicatorMatchRow({ - indexField: newThreatIndicatorRule.indicatorMapping, - indicatorIndexField: newThreatIndicatorRule.indicatorIndexField, + it('Deletes the first row when you have two rows. Both rows valid rows of "index fields" and valid "indicator index fields". The second row should become the first row', () => { + fillIndicatorMatchRow({ + indexField: newThreatIndicatorRule.indicatorMapping, + indicatorIndexField: newThreatIndicatorRule.indicatorIndexField, + }); + getIndicatorAndButton().click(); + fillIndicatorMatchRow({ + rowNumber: 2, + indexField: 'agent.name', + indicatorIndexField: newThreatIndicatorRule.indicatorIndexField, + validColumns: 'indicatorField', + }); + getIndicatorDeleteButton().click(); + getIndicatorIndexComboField().should('have.text', 'agent.name'); + getIndicatorMappingComboField().should( + 'have.text', + newThreatIndicatorRule.indicatorIndexField + ); + getIndicatorIndexComboField(2).should('not.exist'); + getIndicatorMappingComboField(2).should('not.exist'); }); - getIndicatorDeleteButton().click(); - getIndicatorIndexComboField().should('text', 'Search'); - getIndicatorMappingComboField().should('text', 'Search'); - getIndicatorIndexComboField(2).should('not.exist'); - getIndicatorMappingComboField(2).should('not.exist'); - }); - it('Deletes the second row when you have three rows. The first row is valid data, the second row is invalid data, and the third row is valid data. Third row should shift up correctly', () => { - fillIndicatorMatchRow({ - indexField: newThreatIndicatorRule.indicatorMapping, - indicatorIndexField: newThreatIndicatorRule.indicatorIndexField, + it('Deletes the first row when you have two rows. Both rows have valid "index fields" and invalid "indicator index fields". The second row should become the first row', () => { + fillIndicatorMatchRow({ + indexField: newThreatIndicatorRule.indicatorMapping, + indicatorIndexField: 'non-existent-value', + validColumns: 'indexField', + }); + getIndicatorAndButton().click(); + fillIndicatorMatchRow({ + rowNumber: 2, + indexField: newThreatIndicatorRule.indicatorMapping, + indicatorIndexField: 'second-non-existent-value', + validColumns: 'indexField', + }); + getIndicatorDeleteButton().click(); + getIndicatorMappingComboField().should('have.text', 'second-non-existent-value'); + getIndicatorIndexComboField(2).should('not.exist'); + getIndicatorMappingComboField(2).should('not.exist'); }); - getIndicatorAndButton().click(); - fillIndicatorMatchRow({ - rowNumber: 2, - indexField: 'non-existent-value', - indicatorIndexField: 'non-existent-value', - validColumns: 'none', + + it('Deletes the first row when you have two rows. Both rows have valid "indicator index fields" and invalid "index fields". The second row should become the first row', () => { + fillIndicatorMatchRow({ + indexField: 'non-existent-value', + indicatorIndexField: newThreatIndicatorRule.indicatorIndexField, + validColumns: 'indicatorField', + }); + getIndicatorAndButton().click(); + fillIndicatorMatchRow({ + rowNumber: 2, + indexField: 'second-non-existent-value', + indicatorIndexField: newThreatIndicatorRule.indicatorIndexField, + validColumns: 'indicatorField', + }); + getIndicatorDeleteButton().click(); + getIndicatorIndexComboField().should('have.text', 'second-non-existent-value'); + getIndicatorIndexComboField(2).should('not.exist'); + getIndicatorMappingComboField(2).should('not.exist'); }); - getIndicatorAndButton().click(); - fillIndicatorMatchRow({ - rowNumber: 3, - indexField: newThreatIndicatorRule.indicatorMapping, - indicatorIndexField: newThreatIndicatorRule.indicatorIndexField, + + it('Deletes the first row of data but not the UI elements and the text defaults back to the placeholder of Search', () => { + fillIndicatorMatchRow({ + indexField: newThreatIndicatorRule.indicatorMapping, + indicatorIndexField: newThreatIndicatorRule.indicatorIndexField, + }); + getIndicatorDeleteButton().click(); + getIndicatorIndexComboField().should('text', 'Search'); + getIndicatorMappingComboField().should('text', 'Search'); + getIndicatorIndexComboField(2).should('not.exist'); + getIndicatorMappingComboField(2).should('not.exist'); }); - getIndicatorDeleteButton(2).click(); - getIndicatorIndexComboField(1).should('text', newThreatIndicatorRule.indicatorMapping); - getIndicatorMappingComboField(1).should('text', newThreatIndicatorRule.indicatorIndexField); - getIndicatorIndexComboField(2).should('text', newThreatIndicatorRule.indicatorMapping); - getIndicatorMappingComboField(2).should('text', newThreatIndicatorRule.indicatorIndexField); - getIndicatorIndexComboField(3).should('not.exist'); - getIndicatorMappingComboField(3).should('not.exist'); - }); - it('Can add two OR rows and delete the second row. The first row has invalid data and the second row has valid data. The first row is deleted and the second row shifts up correctly.', () => { - fillIndicatorMatchRow({ - indexField: 'non-existent-value-one', - indicatorIndexField: 'non-existent-value-two', - validColumns: 'none', + it('Deletes the second row when you have three rows. The first row is valid data, the second row is invalid data, and the third row is valid data. Third row should shift up correctly', () => { + fillIndicatorMatchRow({ + indexField: newThreatIndicatorRule.indicatorMapping, + indicatorIndexField: newThreatIndicatorRule.indicatorIndexField, + }); + getIndicatorAndButton().click(); + fillIndicatorMatchRow({ + rowNumber: 2, + indexField: 'non-existent-value', + indicatorIndexField: 'non-existent-value', + validColumns: 'none', + }); + getIndicatorAndButton().click(); + fillIndicatorMatchRow({ + rowNumber: 3, + indexField: newThreatIndicatorRule.indicatorMapping, + indicatorIndexField: newThreatIndicatorRule.indicatorIndexField, + }); + getIndicatorDeleteButton(2).click(); + getIndicatorIndexComboField(1).should('text', newThreatIndicatorRule.indicatorMapping); + getIndicatorMappingComboField(1).should( + 'text', + newThreatIndicatorRule.indicatorIndexField + ); + getIndicatorIndexComboField(2).should('text', newThreatIndicatorRule.indicatorMapping); + getIndicatorMappingComboField(2).should( + 'text', + newThreatIndicatorRule.indicatorIndexField + ); + getIndicatorIndexComboField(3).should('not.exist'); + getIndicatorMappingComboField(3).should('not.exist'); }); - getIndicatorOrButton().click(); - fillIndicatorMatchRow({ - rowNumber: 2, - indexField: newThreatIndicatorRule.indicatorMapping, - indicatorIndexField: newThreatIndicatorRule.indicatorIndexField, + + it('Can add two OR rows and delete the second row. The first row has invalid data and the second row has valid data. The first row is deleted and the second row shifts up correctly.', () => { + fillIndicatorMatchRow({ + indexField: 'non-existent-value-one', + indicatorIndexField: 'non-existent-value-two', + validColumns: 'none', + }); + getIndicatorOrButton().click(); + fillIndicatorMatchRow({ + rowNumber: 2, + indexField: newThreatIndicatorRule.indicatorMapping, + indicatorIndexField: newThreatIndicatorRule.indicatorIndexField, + }); + getIndicatorDeleteButton().click(); + getIndicatorIndexComboField().should('text', newThreatIndicatorRule.indicatorMapping); + getIndicatorMappingComboField().should( + 'text', + newThreatIndicatorRule.indicatorIndexField + ); + getIndicatorIndexComboField(2).should('not.exist'); + getIndicatorMappingComboField(2).should('not.exist'); }); - getIndicatorDeleteButton().click(); - getIndicatorIndexComboField().should('text', newThreatIndicatorRule.indicatorMapping); - getIndicatorMappingComboField().should('text', newThreatIndicatorRule.indicatorIndexField); - getIndicatorIndexComboField(2).should('not.exist'); - getIndicatorMappingComboField(2).should('not.exist'); }); }); - }); - describe('Generating signals', () => { - beforeEach(() => { - cleanKibana(); - loginAndWaitForPageWithoutDateRange(DETECTIONS_URL); - waitForAlertsPanelToBeLoaded(); - waitForAlertsIndexToBeCreated(); - goToManageAlertsDetectionRules(); - waitForLoadElasticPrebuiltDetectionRulesTableToBeLoaded(); - goToCreateNewRule(); - selectIndicatorMatchType(); - }); + describe('Generating signals', () => { + beforeEach(() => { + cleanKibana(); + loginAndWaitForPageWithoutDateRange(DETECTIONS_URL); + waitForAlertsPanelToBeLoaded(); + waitForAlertsIndexToBeCreated(); + goToManageAlertsDetectionRules(); + waitForLoadElasticPrebuiltDetectionRulesTableToBeLoaded(); + goToCreateNewRule(); + selectIndicatorMatchType(); + }); - it('Creates and activates a new Indicator Match rule', () => { - fillDefineIndicatorMatchRuleAndContinue(newThreatIndicatorRule); - fillAboutRuleAndContinue(newThreatIndicatorRule); - fillScheduleRuleAndContinue(newThreatIndicatorRule); - createAndActivateRule(); + it('Creates and activates a new Indicator Match rule', () => { + fillDefineIndicatorMatchRuleAndContinue(newThreatIndicatorRule); + fillAboutRuleAndContinue(newThreatIndicatorRule); + fillScheduleRuleAndContinue(newThreatIndicatorRule); + createAndActivateRule(); - cy.get(CUSTOM_RULES_BTN).should('have.text', 'Custom rules (1)'); + cy.get(CUSTOM_RULES_BTN).should('have.text', 'Custom rules (1)'); - changeToThreeHundredRowsPerPage(); - waitForRulesToBeLoaded(); + changeToThreeHundredRowsPerPage(); + waitForRulesToBeLoaded(); - cy.get(RULES_TABLE).then(($table) => { - cy.wrap($table.find(RULES_ROW).length).should('eql', expectedNumberOfRules); - }); + cy.get(RULES_TABLE).then(($table) => { + cy.wrap($table.find(RULES_ROW).length).should('eql', expectedNumberOfRules); + }); - filterByCustomRules(); + filterByCustomRules(); - cy.get(RULES_TABLE).then(($table) => { - cy.wrap($table.find(RULES_ROW).length).should('eql', 1); - }); - cy.get(RULE_NAME).should('have.text', newThreatIndicatorRule.name); - cy.get(RISK_SCORE).should('have.text', newThreatIndicatorRule.riskScore); - cy.get(SEVERITY).should('have.text', newThreatIndicatorRule.severity); - cy.get(RULE_SWITCH).should('have.attr', 'aria-checked', 'true'); - - goToRuleDetails(); - - cy.get(RULE_NAME_HEADER).should('have.text', `${newThreatIndicatorRule.name}`); - cy.get(ABOUT_RULE_DESCRIPTION).should('have.text', newThreatIndicatorRule.description); - cy.get(ABOUT_DETAILS).within(() => { - getDetails(SEVERITY_DETAILS).should('have.text', newThreatIndicatorRule.severity); - getDetails(RISK_SCORE_DETAILS).should('have.text', newThreatIndicatorRule.riskScore); - getDetails(REFERENCE_URLS_DETAILS).should((details) => { - expect(removeExternalLinkText(details.text())).equal(expectedUrls); + cy.get(RULES_TABLE).then(($table) => { + cy.wrap($table.find(RULES_ROW).length).should('eql', 1); }); - getDetails(FALSE_POSITIVES_DETAILS).should('have.text', expectedFalsePositives); - getDetails(MITRE_ATTACK_DETAILS).should((mitre) => { - expect(removeExternalLinkText(mitre.text())).equal(expectedMitre); + cy.get(RULE_NAME).should('have.text', newThreatIndicatorRule.name); + cy.get(RISK_SCORE).should('have.text', newThreatIndicatorRule.riskScore); + cy.get(SEVERITY).should('have.text', newThreatIndicatorRule.severity); + cy.get(RULE_SWITCH).should('have.attr', 'aria-checked', 'true'); + + goToRuleDetails(); + + cy.get(RULE_NAME_HEADER).should('have.text', `${newThreatIndicatorRule.name}`); + cy.get(ABOUT_RULE_DESCRIPTION).should('have.text', newThreatIndicatorRule.description); + cy.get(ABOUT_DETAILS).within(() => { + getDetails(SEVERITY_DETAILS).should('have.text', newThreatIndicatorRule.severity); + getDetails(RISK_SCORE_DETAILS).should('have.text', newThreatIndicatorRule.riskScore); + getDetails(REFERENCE_URLS_DETAILS).should((details) => { + expect(removeExternalLinkText(details.text())).equal(expectedUrls); + }); + getDetails(FALSE_POSITIVES_DETAILS).should('have.text', expectedFalsePositives); + getDetails(MITRE_ATTACK_DETAILS).should((mitre) => { + expect(removeExternalLinkText(mitre.text())).equal(expectedMitre); + }); + getDetails(TAGS_DETAILS).should('have.text', expectedTags); + }); + cy.get(INVESTIGATION_NOTES_TOGGLE).click({ force: true }); + cy.get(ABOUT_INVESTIGATION_NOTES).should('have.text', INVESTIGATION_NOTES_MARKDOWN); + + cy.get(DEFINITION_DETAILS).within(() => { + getDetails(INDEX_PATTERNS_DETAILS).should( + 'have.text', + newThreatIndicatorRule.index!.join('') + ); + getDetails(CUSTOM_QUERY_DETAILS).should('have.text', '*:*'); + getDetails(RULE_TYPE_DETAILS).should('have.text', 'Indicator Match'); + getDetails(TIMELINE_TEMPLATE_DETAILS).should('have.text', 'None'); + getDetails(INDICATOR_INDEX_PATTERNS).should( + 'have.text', + newThreatIndicatorRule.indicatorIndexPattern.join('') + ); + getDetails(INDICATOR_MAPPING).should( + 'have.text', + `${newThreatIndicatorRule.indicatorMapping} MATCHES ${newThreatIndicatorRule.indicatorIndexField}` + ); + getDetails(INDICATOR_INDEX_QUERY).should('have.text', '*:*'); }); - getDetails(TAGS_DETAILS).should('have.text', expectedTags); - }); - cy.get(INVESTIGATION_NOTES_TOGGLE).click({ force: true }); - cy.get(ABOUT_INVESTIGATION_NOTES).should('have.text', INVESTIGATION_NOTES_MARKDOWN); - - cy.get(DEFINITION_DETAILS).within(() => { - getDetails(INDEX_PATTERNS_DETAILS).should( - 'have.text', - newThreatIndicatorRule.index!.join('') - ); - getDetails(CUSTOM_QUERY_DETAILS).should('have.text', '*:*'); - getDetails(RULE_TYPE_DETAILS).should('have.text', 'Indicator Match'); - getDetails(TIMELINE_TEMPLATE_DETAILS).should('have.text', 'None'); - getDetails(INDICATOR_INDEX_PATTERNS).should( - 'have.text', - newThreatIndicatorRule.indicatorIndexPattern.join('') - ); - getDetails(INDICATOR_MAPPING).should( - 'have.text', - `${newThreatIndicatorRule.indicatorMapping} MATCHES ${newThreatIndicatorRule.indicatorIndexField}` - ); - getDetails(INDICATOR_INDEX_QUERY).should('have.text', '*:*'); - }); - cy.get(SCHEDULE_DETAILS).within(() => { - getDetails(RUNS_EVERY_DETAILS).should( - 'have.text', - `${newThreatIndicatorRule.runsEvery.interval}${newThreatIndicatorRule.runsEvery.type}` - ); - getDetails(ADDITIONAL_LOOK_BACK_DETAILS).should( - 'have.text', - `${newThreatIndicatorRule.lookBack.interval}${newThreatIndicatorRule.lookBack.type}` - ); - }); + cy.get(SCHEDULE_DETAILS).within(() => { + getDetails(RUNS_EVERY_DETAILS).should( + 'have.text', + `${newThreatIndicatorRule.runsEvery.interval}${newThreatIndicatorRule.runsEvery.type}` + ); + getDetails(ADDITIONAL_LOOK_BACK_DETAILS).should( + 'have.text', + `${newThreatIndicatorRule.lookBack.interval}${newThreatIndicatorRule.lookBack.type}` + ); + }); - waitForTheRuleToBeExecuted(); - waitForAlertsToPopulate(); - - cy.get(NUMBER_OF_ALERTS).should('have.text', expectedNumberOfAlerts); - cy.get(ALERT_RULE_NAME).first().should('have.text', newThreatIndicatorRule.name); - cy.get(ALERT_RULE_VERSION).first().should('have.text', '1'); - cy.get(ALERT_RULE_METHOD).first().should('have.text', 'threat_match'); - cy.get(ALERT_RULE_SEVERITY) - .first() - .should('have.text', newThreatIndicatorRule.severity.toLowerCase()); - cy.get(ALERT_RULE_RISK_SCORE).first().should('have.text', newThreatIndicatorRule.riskScore); + waitForTheRuleToBeExecuted(); + waitForAlertsToPopulate(); + + cy.get(NUMBER_OF_ALERTS).should('have.text', expectedNumberOfAlerts); + cy.get(ALERT_RULE_NAME).first().should('have.text', newThreatIndicatorRule.name); + cy.get(ALERT_RULE_VERSION).first().should('have.text', '1'); + cy.get(ALERT_RULE_METHOD).first().should('have.text', 'threat_match'); + cy.get(ALERT_RULE_SEVERITY) + .first() + .should('have.text', newThreatIndicatorRule.severity.toLowerCase()); + cy.get(ALERT_RULE_RISK_SCORE).first().should('have.text', newThreatIndicatorRule.riskScore); + }); }); }); }); diff --git a/x-pack/plugins/security_solution/cypress/integration/detection_rules/threshold_rule.spec.ts b/x-pack/plugins/security_solution/cypress/integration/detection_rules/threshold_rule.spec.ts index d93ed7a0e97a5..3c188345111c8 100644 --- a/x-pack/plugins/security_solution/cypress/integration/detection_rules/threshold_rule.spec.ts +++ b/x-pack/plugins/security_solution/cypress/integration/detection_rules/threshold_rule.spec.ts @@ -80,101 +80,104 @@ import { loginAndWaitForPageWithoutDateRange } from '../../tasks/login'; import { DETECTIONS_URL } from '../../urls/navigation'; -describe('Detection rules, threshold', () => { - const expectedUrls = newThresholdRule.referenceUrls.join(''); - const expectedFalsePositives = newThresholdRule.falsePositivesExamples.join(''); - const expectedTags = newThresholdRule.tags.join(''); - const expectedMitre = formatMitreAttackDescription(newThresholdRule.mitre); - - const rule = { ...newThresholdRule }; - - beforeEach(() => { - cleanKibana(); - createTimeline(newThresholdRule.timeline).then((response) => { - rule.timeline.id = response.body.data.persistTimeline.timeline.savedObjectId; +// Skipped until post-FF for 7.12 +describe.skip('Threshold Rules', () => { + describe('Detection rules, threshold', () => { + const expectedUrls = newThresholdRule.referenceUrls.join(''); + const expectedFalsePositives = newThresholdRule.falsePositivesExamples.join(''); + const expectedTags = newThresholdRule.tags.join(''); + const expectedMitre = formatMitreAttackDescription(newThresholdRule.mitre); + + const rule = { ...newThresholdRule }; + + beforeEach(() => { + cleanKibana(); + createTimeline(newThresholdRule.timeline).then((response) => { + rule.timeline.id = response.body.data.persistTimeline.timeline.savedObjectId; + }); }); - }); - it('Creates and activates a new threshold rule', () => { - loginAndWaitForPageWithoutDateRange(DETECTIONS_URL); - waitForAlertsPanelToBeLoaded(); - waitForAlertsIndexToBeCreated(); - goToManageAlertsDetectionRules(); - waitForLoadElasticPrebuiltDetectionRulesTableToBeLoaded(); - goToCreateNewRule(); - selectThresholdRuleType(); - fillDefineThresholdRuleAndContinue(rule); - fillAboutRuleAndContinue(rule); - fillScheduleRuleAndContinue(rule); - createAndActivateRule(); - - cy.get(CUSTOM_RULES_BTN).should('have.text', 'Custom rules (1)'); - - changeToThreeHundredRowsPerPage(); - waitForRulesToBeLoaded(); - - const expectedNumberOfRules = 1; - cy.get(RULES_TABLE).then(($table) => { - cy.wrap($table.find(RULES_ROW).length).should('eql', expectedNumberOfRules); - }); + it('Creates and activates a new threshold rule', () => { + loginAndWaitForPageWithoutDateRange(DETECTIONS_URL); + waitForAlertsPanelToBeLoaded(); + waitForAlertsIndexToBeCreated(); + goToManageAlertsDetectionRules(); + waitForLoadElasticPrebuiltDetectionRulesTableToBeLoaded(); + goToCreateNewRule(); + selectThresholdRuleType(); + fillDefineThresholdRuleAndContinue(rule); + fillAboutRuleAndContinue(rule); + fillScheduleRuleAndContinue(rule); + createAndActivateRule(); + + cy.get(CUSTOM_RULES_BTN).should('have.text', 'Custom rules (1)'); + + changeToThreeHundredRowsPerPage(); + waitForRulesToBeLoaded(); + + const expectedNumberOfRules = 1; + cy.get(RULES_TABLE).then(($table) => { + cy.wrap($table.find(RULES_ROW).length).should('eql', expectedNumberOfRules); + }); - filterByCustomRules(); + filterByCustomRules(); - cy.get(RULES_TABLE).then(($table) => { - cy.wrap($table.find(RULES_ROW).length).should('eql', 1); - }); - cy.get(RULE_NAME).should('have.text', rule.name); - cy.get(RISK_SCORE).should('have.text', rule.riskScore); - cy.get(SEVERITY).should('have.text', rule.severity); - cy.get(RULE_SWITCH).should('have.attr', 'aria-checked', 'true'); - - goToRuleDetails(); - - cy.get(RULE_NAME_HEADER).should('have.text', `${rule.name}`); - cy.get(ABOUT_RULE_DESCRIPTION).should('have.text', rule.description); - cy.get(ABOUT_DETAILS).within(() => { - getDetails(SEVERITY_DETAILS).should('have.text', rule.severity); - getDetails(RISK_SCORE_DETAILS).should('have.text', rule.riskScore); - getDetails(REFERENCE_URLS_DETAILS).should((details) => { - expect(removeExternalLinkText(details.text())).equal(expectedUrls); + cy.get(RULES_TABLE).then(($table) => { + cy.wrap($table.find(RULES_ROW).length).should('eql', 1); }); - getDetails(FALSE_POSITIVES_DETAILS).should('have.text', expectedFalsePositives); - getDetails(MITRE_ATTACK_DETAILS).should((mitre) => { - expect(removeExternalLinkText(mitre.text())).equal(expectedMitre); + cy.get(RULE_NAME).should('have.text', rule.name); + cy.get(RISK_SCORE).should('have.text', rule.riskScore); + cy.get(SEVERITY).should('have.text', rule.severity); + cy.get(RULE_SWITCH).should('have.attr', 'aria-checked', 'true'); + + goToRuleDetails(); + + cy.get(RULE_NAME_HEADER).should('have.text', `${rule.name}`); + cy.get(ABOUT_RULE_DESCRIPTION).should('have.text', rule.description); + cy.get(ABOUT_DETAILS).within(() => { + getDetails(SEVERITY_DETAILS).should('have.text', rule.severity); + getDetails(RISK_SCORE_DETAILS).should('have.text', rule.riskScore); + getDetails(REFERENCE_URLS_DETAILS).should((details) => { + expect(removeExternalLinkText(details.text())).equal(expectedUrls); + }); + getDetails(FALSE_POSITIVES_DETAILS).should('have.text', expectedFalsePositives); + getDetails(MITRE_ATTACK_DETAILS).should((mitre) => { + expect(removeExternalLinkText(mitre.text())).equal(expectedMitre); + }); + getDetails(TAGS_DETAILS).should('have.text', expectedTags); + }); + cy.get(INVESTIGATION_NOTES_TOGGLE).click({ force: true }); + cy.get(ABOUT_INVESTIGATION_NOTES).should('have.text', INVESTIGATION_NOTES_MARKDOWN); + cy.get(DEFINITION_DETAILS).within(() => { + getDetails(INDEX_PATTERNS_DETAILS).should('have.text', indexPatterns.join('')); + getDetails(CUSTOM_QUERY_DETAILS).should('have.text', rule.customQuery); + getDetails(RULE_TYPE_DETAILS).should('have.text', 'Threshold'); + getDetails(TIMELINE_TEMPLATE_DETAILS).should('have.text', 'None'); + getDetails(THRESHOLD_DETAILS).should( + 'have.text', + `Results aggregated by ${rule.thresholdField} >= ${rule.threshold}` + ); + }); + cy.get(SCHEDULE_DETAILS).within(() => { + getDetails(RUNS_EVERY_DETAILS).should( + 'have.text', + `${rule.runsEvery.interval}${rule.runsEvery.type}` + ); + getDetails(ADDITIONAL_LOOK_BACK_DETAILS).should( + 'have.text', + `${rule.lookBack.interval}${rule.lookBack.type}` + ); }); - getDetails(TAGS_DETAILS).should('have.text', expectedTags); - }); - cy.get(INVESTIGATION_NOTES_TOGGLE).click({ force: true }); - cy.get(ABOUT_INVESTIGATION_NOTES).should('have.text', INVESTIGATION_NOTES_MARKDOWN); - cy.get(DEFINITION_DETAILS).within(() => { - getDetails(INDEX_PATTERNS_DETAILS).should('have.text', indexPatterns.join('')); - getDetails(CUSTOM_QUERY_DETAILS).should('have.text', rule.customQuery); - getDetails(RULE_TYPE_DETAILS).should('have.text', 'Threshold'); - getDetails(TIMELINE_TEMPLATE_DETAILS).should('have.text', 'None'); - getDetails(THRESHOLD_DETAILS).should( - 'have.text', - `Results aggregated by ${rule.thresholdField} >= ${rule.threshold}` - ); - }); - cy.get(SCHEDULE_DETAILS).within(() => { - getDetails(RUNS_EVERY_DETAILS).should( - 'have.text', - `${rule.runsEvery.interval}${rule.runsEvery.type}` - ); - getDetails(ADDITIONAL_LOOK_BACK_DETAILS).should( - 'have.text', - `${rule.lookBack.interval}${rule.lookBack.type}` - ); - }); - waitForTheRuleToBeExecuted(); - waitForAlertsToPopulate(); + waitForTheRuleToBeExecuted(); + waitForAlertsToPopulate(); - cy.get(NUMBER_OF_ALERTS).should(($count) => expect(+$count.text()).to.be.lt(100)); - cy.get(ALERT_RULE_NAME).first().should('have.text', rule.name); - cy.get(ALERT_RULE_VERSION).first().should('have.text', '1'); - cy.get(ALERT_RULE_METHOD).first().should('have.text', 'threshold'); - cy.get(ALERT_RULE_SEVERITY).first().should('have.text', rule.severity.toLowerCase()); - cy.get(ALERT_RULE_RISK_SCORE).first().should('have.text', rule.riskScore); + cy.get(NUMBER_OF_ALERTS).should(($count) => expect(+$count.text()).to.be.lt(100)); + cy.get(ALERT_RULE_NAME).first().should('have.text', rule.name); + cy.get(ALERT_RULE_VERSION).first().should('have.text', '1'); + cy.get(ALERT_RULE_METHOD).first().should('have.text', 'threshold'); + cy.get(ALERT_RULE_SEVERITY).first().should('have.text', rule.severity.toLowerCase()); + cy.get(ALERT_RULE_RISK_SCORE).first().should('have.text', rule.riskScore); + }); }); }); diff --git a/x-pack/plugins/security_solution/public/common/components/matrix_histogram/types.ts b/x-pack/plugins/security_solution/public/common/components/matrix_histogram/types.ts index 0da881d2c6ea4..b993bcda56b8e 100644 --- a/x-pack/plugins/security_solution/public/common/components/matrix_histogram/types.ts +++ b/x-pack/plugins/security_solution/public/common/components/matrix_histogram/types.ts @@ -74,7 +74,14 @@ export interface MatrixHistogramQueryProps { stackByField: string; startDate: string; histogramType: MatrixHistogramType; - threshold?: { field: string | undefined; value: number } | undefined; + threshold?: + | { + field: string | string[] | undefined; + value: number; + cardinality_field?: string | undefined; + cardinality_value?: number | undefined; + } + | undefined; skip?: boolean; isPtrIncluded?: boolean; } diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx b/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx index 14ccae250ac48..9f5ab7be8a117 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx @@ -8,7 +8,7 @@ /* eslint-disable complexity */ import dateMath from '@elastic/datemath'; -import { get, getOr, isEmpty, find } from 'lodash/fp'; +import { getOr, isEmpty } from 'lodash/fp'; import moment from 'moment'; import { i18n } from '@kbn/i18n'; @@ -38,7 +38,10 @@ import { replaceTemplateFieldFromDataProviders, } from './helpers'; import { KueryFilterQueryKind } from '../../../common/store'; -import { DataProvider } from '../../../timelines/components/timeline/data_providers/data_provider'; +import { + DataProvider, + QueryOperator, +} from '../../../timelines/components/timeline/data_providers/data_provider'; import { esFilters } from '../../../../../../../src/plugins/data/public'; export const getUpdateAlertsQuery = (eventIds: Readonly) => { @@ -47,7 +50,7 @@ export const getUpdateAlertsQuery = (eventIds: Readonly) => { bool: { filter: { terms: { - _id: [...eventIds], + _id: eventIds, }, }, }, @@ -148,35 +151,76 @@ export const getThresholdAggregationDataProvider = ( nonEcsData: TimelineNonEcsData[] ): DataProvider[] => { const thresholdEcsData: Ecs[] = Array.isArray(ecsData) ? ecsData : [ecsData]; - return thresholdEcsData.reduce((acc, tresholdData) => { - const aggregationField = tresholdData.signal?.rule?.threshold?.field!; - const aggregationValue = - get(aggregationField, tresholdData) ?? find(['field', aggregationField], nonEcsData)?.value; - const dataProviderValue = Array.isArray(aggregationValue) - ? aggregationValue[0] - : aggregationValue; - - if (!dataProviderValue) { - return acc; + return thresholdEcsData.reduce((outerAcc, thresholdData) => { + const threshold = thresholdData.signal?.rule?.threshold as string[]; + + let aggField: string[] = []; + let thresholdResult: { + terms?: Array<{ + field?: string; + value: string; + }>; + count: number; + }; + + try { + thresholdResult = JSON.parse((thresholdData.signal?.threshold_result as string[])[0]); + aggField = JSON.parse(threshold[0]).field; + } catch (err) { + thresholdResult = { + terms: [ + { + field: (thresholdData.rule?.threshold as { field: string }).field, + value: (thresholdData.signal?.threshold_result as { value: string }).value, + }, + ], + count: (thresholdData.signal?.threshold_result as { count: number }).count, + }; } - const aggregationFieldId = aggregationField.replace('.', '-'); + const aggregationFields = Array.isArray(aggField) ? aggField : [aggField]; return [ - ...acc, - { - and: [], - id: `send-alert-to-timeline-action-default-draggable-event-details-value-formatted-field-value-${TimelineId.active}-${aggregationFieldId}-${dataProviderValue}`, - name: aggregationField, - enabled: true, - excluded: false, - kqlQuery: '', - queryMatch: { - field: aggregationField, - value: dataProviderValue, - operator: ':', - }, - }, + ...outerAcc, + ...aggregationFields.reduce((acc, aggregationField, i) => { + const aggregationValue = (thresholdResult.terms ?? []).filter( + (term: { field?: string | undefined; value: string }) => term.field === aggregationField + )[0].value; + const dataProviderValue = Array.isArray(aggregationValue) + ? aggregationValue[0] + : aggregationValue; + + if (!dataProviderValue) { + return acc; + } + + const aggregationFieldId = aggregationField.replace('.', '-'); + const dataProviderPartial = { + id: `send-alert-to-timeline-action-default-draggable-event-details-value-formatted-field-value-${TimelineId.active}-${aggregationFieldId}-${dataProviderValue}`, + name: aggregationField, + enabled: true, + excluded: false, + kqlQuery: '', + queryMatch: { + field: aggregationField, + value: dataProviderValue, + operator: ':' as QueryOperator, + }, + }; + + if (i === 0) { + return [ + ...acc, + { + ...dataProviderPartial, + and: [], + }, + ]; + } else { + acc[0].and.push(dataProviderPartial); + return acc; + } + }, []), ]; }, []); }; @@ -409,7 +453,7 @@ export const sendAlertToTimelineAction = async ({ ...timelineDefaults, description: `_id: ${ecsData._id}`, filters: getFiltersFromRule(ecsData.signal?.rule?.filters as string[]), - dataProviders: [...getThresholdAggregationDataProvider(ecs, nonEcsData)], + dataProviders: getThresholdAggregationDataProvider(ecsData, nonEcsData), id: TimelineId.active, indexNames: [], dateRange: { diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/index.test.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/index.test.tsx index e954f961e7336..bb87242d9bf10 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/index.test.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/index.test.tsx @@ -288,7 +288,12 @@ describe('PreviewQuery', () => { idAria="queryPreview" query={{ query: { query: 'file where true', language: 'kuery' }, filters: [] }} index={['foo-*']} - threshold={{ field: 'agent.hostname', value: 200 }} + threshold={{ + field: 'agent.hostname', + value: 200, + cardinality_field: 'user.name', + cardinality_value: 2, + }} isDisabled={false} /> @@ -330,7 +335,12 @@ describe('PreviewQuery', () => { idAria="queryPreview" query={{ query: { query: 'file where true', language: 'kuery' }, filters: [] }} index={['foo-*']} - threshold={{ field: 'agent.hostname', value: 200 }} + threshold={{ + field: 'agent.hostname', + value: 200, + cardinality_field: 'user.name', + cardinality_value: 2, + }} isDisabled={false} /> @@ -369,7 +379,12 @@ describe('PreviewQuery', () => { idAria="queryPreview" query={{ query: { query: 'file where true', language: 'kuery' }, filters: [] }} index={['foo-*']} - threshold={{ field: undefined, value: 200 }} + threshold={{ + field: undefined, + value: 200, + cardinality_field: 'user.name', + cardinality_value: 2, + }} isDisabled={false} /> @@ -396,7 +411,12 @@ describe('PreviewQuery', () => { idAria="queryPreview" query={{ query: { query: 'file where true', language: 'kuery' }, filters: [] }} index={['foo-*']} - threshold={{ field: ' ', value: 200 }} + threshold={{ + field: ' ', + value: 200, + cardinality_field: 'user.name', + cardinality_value: 2, + }} isDisabled={false} /> diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/index.tsx index 6a11dcf316de3..377259fc9b212 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/index.tsx @@ -56,7 +56,14 @@ export const initialState: State = { showNonEqlHistogram: false, }; -export type Threshold = { field: string | undefined; value: number } | undefined; +export type Threshold = + | { + field: string | string[] | undefined; + value: number; + cardinality_field: string | undefined; + cardinality_value: number | undefined; + } + | undefined; interface PreviewQueryProps { dataTestSubj: string; diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/reducer.test.ts b/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/reducer.test.ts index ff90978dbb53f..d1a9e5c5f768f 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/reducer.test.ts +++ b/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/reducer.test.ts @@ -334,7 +334,12 @@ describe('queryPreviewReducer', () => { test('should set thresholdFieldExists to true if threshold field is defined and not empty string', () => { const update = reducer(initialState, { type: 'setThresholdQueryVals', - threshold: { field: 'agent.hostname', value: 200 }, + threshold: { + field: 'agent.hostname', + value: 200, + cardinality_field: 'user.name', + cardinality_value: 2, + }, ruleType: 'threshold', }); @@ -347,7 +352,12 @@ describe('queryPreviewReducer', () => { test('should set thresholdFieldExists to false if threshold field is not defined', () => { const update = reducer(initialState, { type: 'setThresholdQueryVals', - threshold: { field: undefined, value: 200 }, + threshold: { + field: undefined, + value: 200, + cardinality_field: 'user.name', + cardinality_value: 2, + }, ruleType: 'threshold', }); @@ -360,7 +370,12 @@ describe('queryPreviewReducer', () => { test('should set thresholdFieldExists to false if threshold field is empty string', () => { const update = reducer(initialState, { type: 'setThresholdQueryVals', - threshold: { field: ' ', value: 200 }, + threshold: { + field: ' ', + value: 200, + cardinality_field: 'user.name', + cardinality_value: 2, + }, ruleType: 'threshold', }); @@ -373,7 +388,12 @@ describe('queryPreviewReducer', () => { test('should set showNonEqlHistogram to false if ruleType is eql', () => { const update = reducer(initialState, { type: 'setThresholdQueryVals', - threshold: { field: 'agent.hostname', value: 200 }, + threshold: { + field: 'agent.hostname', + value: 200, + cardinality_field: 'user.name', + cardinality_value: 2, + }, ruleType: 'eql', }); @@ -385,7 +405,12 @@ describe('queryPreviewReducer', () => { test('should set showNonEqlHistogram to true if ruleType is query', () => { const update = reducer(initialState, { type: 'setThresholdQueryVals', - threshold: { field: 'agent.hostname', value: 200 }, + threshold: { + field: 'agent.hostname', + value: 200, + cardinality_field: 'user.name', + cardinality_value: 2, + }, ruleType: 'query', }); @@ -397,7 +422,12 @@ describe('queryPreviewReducer', () => { test('should set showNonEqlHistogram to true if ruleType is saved_query', () => { const update = reducer(initialState, { type: 'setThresholdQueryVals', - threshold: { field: 'agent.hostname', value: 200 }, + threshold: { + field: 'agent.hostname', + value: 200, + cardinality_field: 'user.name', + cardinality_value: 2, + }, ruleType: 'saved_query', }); diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/reducer.ts b/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/reducer.ts index 84206b51272df..2d301bf96122d 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/reducer.ts +++ b/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/reducer.ts @@ -67,6 +67,7 @@ export type Action = type: 'setToFrom'; }; +/* eslint-disable-next-line complexity */ export const queryPreviewReducer = () => (state: State, action: Action): State => { switch (action.type) { case 'setQueryInfo': { @@ -131,7 +132,9 @@ export const queryPreviewReducer = () => (state: State, action: Action): State = const thresholdField = action.threshold != null && action.threshold.field != null && - action.threshold.field.trim() !== ''; + ((typeof action.threshold.field === 'string' && action.threshold.field.trim() !== '') || + (Array.isArray(action.threshold.field) && + action.threshold.field.every((field) => field.trim() !== ''))); const showNonEqlHist = action.ruleType === 'query' || action.ruleType === 'saved_query' || diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx index 5613d9bc1b458..4c7a34dbdf080 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx @@ -82,6 +82,8 @@ const stepDefineDefaultValue: DefineStepRule = { threshold: { field: [], value: '200', + cardinality_field: [], + cardinality_value: '2', }, timeline: { id: null, @@ -150,17 +152,30 @@ const StepDefineRuleComponent: FC = ({ ruleType: formRuleType, queryBar: formQuery, threatIndex: formThreatIndex, - 'threshold.value': formThresholdValue, 'threshold.field': formThresholdField, + 'threshold.value': formThresholdValue, + 'threshold.cardinality_field': formThresholdCardinalityField, + 'threshold.cardinality_value': formThresholdCardinalityValue, }, ] = useFormData< DefineStepRule & { - 'threshold.value': number | undefined; 'threshold.field': string[] | undefined; + 'threshold.value': number | undefined; + 'threshold.cardinality_field': string[] | undefined; + 'threshold.cardinality_value': number | undefined; } >({ form, - watch: ['index', 'ruleType', 'queryBar', 'threshold.value', 'threshold.field', 'threatIndex'], + watch: [ + 'index', + 'ruleType', + 'queryBar', + 'threshold.field', + 'threshold.value', + 'threshold.cardinality_field', + 'threshold.cardinality_value', + 'threatIndex', + ], }); const [isQueryBarValid, setIsQueryBarValid] = useState(false); const index = formIndex || initialState.index; @@ -274,17 +289,32 @@ const StepDefineRuleComponent: FC = ({ }, []); const thresholdFormValue = useMemo((): Threshold | undefined => { - return formThresholdValue != null && formThresholdField != null - ? { value: formThresholdValue, field: formThresholdField[0] } + return formThresholdValue != null && + formThresholdField != null && + formThresholdCardinalityField != null && + formThresholdCardinalityValue != null + ? { + field: formThresholdField[0], + value: formThresholdValue, + cardinality_field: formThresholdCardinalityField[0], + cardinality_value: formThresholdCardinalityValue, + } : undefined; - }, [formThresholdField, formThresholdValue]); + }, [ + formThresholdField, + formThresholdValue, + formThresholdCardinalityField, + formThresholdCardinalityValue, + ]); const ThresholdInputChildren = useCallback( - ({ thresholdField, thresholdValue }) => ( + ({ thresholdField, thresholdValue, thresholdCardinalityField, thresholdCardinalityValue }) => ( ), [aggregatableFields] @@ -429,6 +459,12 @@ const StepDefineRuleComponent: FC = ({ thresholdValue: { path: 'threshold.value', }, + thresholdCardinalityField: { + path: 'threshold.cardinality_field', + }, + thresholdCardinalityValue: { + path: 'threshold.cardinality_value', + }, }} > {ThresholdInputChildren} diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx index 18f5008ff05f7..a5352ede83d51 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx @@ -197,13 +197,13 @@ export const schema: FormSchema = { label: i18n.translate( 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThresholdFieldLabel', { - defaultMessage: 'Field', + defaultMessage: 'Group by', } ), helpText: i18n.translate( 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThresholdFieldHelpText', { - defaultMessage: 'Select a field to group results by', + defaultMessage: "Select fields to group by. Fields are joined together with 'AND'", } ), }, @@ -239,6 +239,53 @@ export const schema: FormSchema = { }, ], }, + cardinality_field: { + type: FIELD_TYPES.COMBO_BOX, + label: i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThresholdCardinalityFieldLabel', + { + defaultMessage: 'Count', + } + ), + helpText: i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThresholdFieldCardinalityFieldHelpText', + { + defaultMessage: 'Select a field to check cardinality', + } + ), + }, + cardinality_value: { + type: FIELD_TYPES.NUMBER, + label: i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThresholdCardinalityValueFieldLabel', + { + defaultMessage: 'Unique values', + } + ), + validations: [ + { + validator: ( + ...args: Parameters + ): ReturnType> | undefined => { + const [{ formData }] = args; + const needsValidation = isThresholdRule(formData.ruleType); + if (!needsValidation) { + return; + } + return fieldValidators.numberGreaterThanField({ + than: 1, + message: i18n.translate( + 'xpack.securitySolution.detectionEngine.validations.thresholdValueFieldData.numberGreaterThanOrEqualOneErrorMessage', + { + defaultMessage: 'Value must be greater than or equal to one.', + } + ), + allowEquality: true, + })(...args); + }, + }, + ], + }, }, threatIndex: { type: FIELD_TYPES.COMBO_BOX, diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/threshold_input/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/threshold_input/index.tsx index d473a83a9e35e..287c99dce3e60 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/threshold_input/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/threshold_input/index.tsx @@ -19,11 +19,15 @@ const FIELD_COMBO_BOX_WIDTH = 410; export interface FieldValueThreshold { field: string[]; value: string; + cardinality_field: string[]; + cardinality_value: string; } interface ThresholdInputProps { thresholdField: FieldHook; thresholdValue: FieldHook; + thresholdCardinalityField: FieldHook; + thresholdCardinalityValue: FieldHook; browserFields: BrowserFields; } @@ -33,16 +37,19 @@ const OperatorWrapper = styled(EuiFlexItem)` const fieldDescribedByIds = ['detectionEngineStepDefineRuleThresholdField']; const valueDescribedByIds = ['detectionEngineStepDefineRuleThresholdValue']; +const cardinalityFieldDescribedByIds = ['detectionEngineStepDefineRuleThresholdCardinalityField']; +const cardinalityValueDescribedByIds = ['detectionEngineStepDefineRuleThresholdCardinalityValue']; const ThresholdInputComponent: React.FC = ({ thresholdField, thresholdValue, browserFields, + thresholdCardinalityField, + thresholdCardinalityValue, }: ThresholdInputProps) => { const fieldEuiFieldProps = useMemo( () => ({ fullWidth: true, - singleSelection: { asPlainText: true }, noSuggestions: false, options: getCategorizedFieldNames(browserFields), placeholder: THRESHOLD_FIELD_PLACEHOLDER, @@ -51,29 +58,65 @@ const ThresholdInputComponent: React.FC = ({ }), [browserFields] ); + const cardinalityFieldEuiProps = useMemo( + () => ({ + fullWidth: true, + noSuggestions: false, + options: getCategorizedFieldNames(browserFields), + placeholder: THRESHOLD_FIELD_PLACEHOLDER, + onCreateOption: undefined, + style: { width: `${FIELD_COMBO_BOX_WIDTH}px` }, + singleSelection: { asPlainText: true }, + }), + [browserFields] + ); return ( - - - - - {'>='} - - - + + + + + + {'>='} + + + + + + + + + {'>='} + + + + ); }; diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/__mocks__/mock.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/__mocks__/mock.ts index 9853b8c6d34bc..8bdbb1a74c73a 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/__mocks__/mock.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/__mocks__/mock.ts @@ -141,8 +141,10 @@ export const mockRuleWithEverything = (id: string): Rule => ({ type: 'saved_query', threat: getThreatMock(), threshold: { - field: 'host.name', + field: ['host.name'], value: 50, + cardinality_field: ['process.name'], + cardinality_value: 2, }, throttle: 'no_actions', timestamp_override: 'event.ingested', @@ -192,6 +194,8 @@ export const mockDefineStepRule = (): DefineStepRule => ({ threshold: { field: [''], value: '100', + cardinality_field: [''], + cardinality_value: '2', }, }); diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts index 7c447214cfdeb..12e6d276c18d8 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts @@ -219,8 +219,10 @@ export const formatDefineStepData = (defineStepData: DefineStepRule): DefineStep saved_id: ruleFields.queryBar?.saved_id, ...(ruleType === 'threshold' && { threshold: { - field: ruleFields.threshold?.field[0] ?? '', + field: ruleFields.threshold?.field ?? [], value: parseInt(ruleFields.threshold?.value, 10) ?? 0, + cardinality_field: ruleFields.threshold.cardinality_field[0] ?? '', + cardinality_value: parseInt(ruleFields.threshold?.cardinality_value, 10) ?? 0, }, }), } diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx index f0511602bd67f..29d1512030e74 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx @@ -84,6 +84,8 @@ describe('rule helpers', () => { threshold: { field: ['host.name'], value: '50', + cardinality_field: ['process.name'], + cardinality_value: '2', }, threatIndex: [], threatMapping: [], @@ -213,6 +215,8 @@ describe('rule helpers', () => { threshold: { field: [], value: '100', + cardinality_field: [], + cardinality_value: '0', }, threatIndex: [], threatMapping: [], @@ -255,6 +259,8 @@ describe('rule helpers', () => { threshold: { field: [], value: '100', + cardinality_field: [], + cardinality_value: '0', }, threatIndex: [], threatMapping: [], diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx index c862d484b282b..7c3930bb21d9a 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx @@ -99,8 +99,18 @@ export const getDefineStepsData = (rule: Rule): DefineStepRule => ({ title: rule.timeline_title ?? null, }, threshold: { - field: rule.threshold?.field ? [rule.threshold.field] : [], + field: rule.threshold?.field + ? Array.isArray(rule.threshold.field) + ? rule.threshold.field + : [rule.threshold.field] + : [], value: `${rule.threshold?.value || 100}`, + cardinality_field: Array.isArray(rule.threshold?.cardinality_field) + ? rule.threshold!.cardinality_field + : rule.threshold?.cardinality_field != null + ? [rule.threshold!.cardinality_field] + : [], + cardinality_value: `${rule.threshold?.cardinality_value ?? 0}`, }, }); diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts index 94fdcc4069fc2..668ca556539ad 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts @@ -158,8 +158,10 @@ export interface DefineStepRuleJson { query?: string; language?: string; threshold?: { - field: string; + field: string[]; value: number; + cardinality_field: string; + cardinality_value: number; }; threat_query?: string; threat_mapping?: ThreatMapping; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json index 909264c57067b..22dba81e5c8e6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json @@ -359,11 +359,28 @@ }, "threshold_result": { "properties": { + "terms": { + "properties": { + "field": { + "type": "keyword" + }, + "value": { + "type": "keyword" + } + } + }, + "cardinality": { + "properties": { + "field": { + "type": "keyword" + }, + "value": { + "type": "long" + } + } + }, "count": { "type": "long" - }, - "value": { - "type": "keyword" } } }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index 6177fc4cd4661..977b38e59f856 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -374,6 +374,100 @@ export const sampleSignalHit = (): SignalHit => ({ }, }); +export const sampleThresholdSignalHit = (): SignalHit => ({ + '@timestamp': '2020-04-20T21:27:45+0000', + event: { + kind: 'signal', + }, + signal: { + parents: [ + { + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + { + id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], + ancestors: [ + { + id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + { + id: '730ddf9e-5a00-4f85-9ddf-5878ca511a87', + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], + original_time: '2021-02-16T17:37:34.275Z', + status: 'open', + threshold_result: { + count: 72, + terms: [{ field: 'host.name', value: 'a hostname' }], + cardinality: [{ field: 'process.name', value: 6 }], + }, + rule: { + author: [], + id: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9', + created_at: '2020-04-20T21:27:45+0000', + updated_at: '2020-04-20T21:27:45+0000', + created_by: 'elastic', + description: 'some description', + enabled: true, + false_positives: ['false positive 1', 'false positive 2'], + from: 'now-6m', + immutable: false, + name: 'Query with a rule id', + query: 'user.name: root or user.name: admin', + references: ['test 1', 'test 2'], + severity: 'high', + severity_mapping: [], + threshold: { + field: ['host.name'], + value: 5, + cardinality_field: 'process.name', + cardinality_value: 2, + }, + updated_by: 'elastic_kibana', + tags: ['some fake tag 1', 'some fake tag 2'], + to: 'now', + type: 'query', + threat: [], + version: 1, + status: 'succeeded', + status_date: '2020-02-22T16:47:50.047Z', + last_success_at: '2020-02-22T16:47:50.047Z', + last_success_message: 'succeeded', + output_index: '.siem-signals-default', + max_signals: 100, + risk_score: 55, + risk_score_mapping: [], + language: 'kuery', + rule_id: 'query-rule-id', + interval: '5m', + exceptions_list: getListArrayMock(), + }, + depth: 1, + }, +}); + +export const sampleWrappedThresholdSignalHit = (): WrappedSignalHit => { + return { + _index: 'myFakeSignalIndex', + _id: sampleIdGuid, + _source: sampleThresholdSignalHit(), + }; +}; + export const sampleBulkCreateDuplicateResult = { took: 60, errors: true, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts index ca8fa821ce032..362c368881b37 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts @@ -36,7 +36,13 @@ describe('buildBulkBody', () => { delete doc._source.source; const fakeSignalSourceHit = buildBulkBody({ doc, - ruleParams: sampleParams, + ruleParams: { + ...sampleParams, + threshold: { + field: ['host.name'], + value: 100, + }, + }, id: sampleRuleGuid, name: 'rule-name', actions: [], @@ -110,6 +116,10 @@ describe('buildBulkBody', () => { severity_mapping: [], tags: ['some fake tag 1', 'some fake tag 2'], threat: [], + threshold: { + field: ['host.name'], + value: 100, + }, throttle: 'no_actions', type: 'query', to: 'now', @@ -136,15 +146,25 @@ describe('buildBulkBody', () => { _source: { ...baseDoc._source, threshold_result: { + terms: [ + { + value: 'abcd', + }, + ], count: 5, - value: 'abcd', }, }, }; delete doc._source.source; const fakeSignalSourceHit = buildBulkBody({ doc, - ruleParams: sampleParams, + ruleParams: { + ...sampleParams, + threshold: { + field: [], + value: 4, + }, + }, id: sampleRuleGuid, name: 'rule-name', actions: [], @@ -218,6 +238,10 @@ describe('buildBulkBody', () => { severity_mapping: [], tags: ['some fake tag 1', 'some fake tag 2'], threat: [], + threshold: { + field: [], + value: 4, + }, throttle: 'no_actions', type: 'query', to: 'now', @@ -231,8 +255,12 @@ describe('buildBulkBody', () => { exceptions_list: getListArrayMock(), }, threshold_result: { + terms: [ + { + value: 'abcd', + }, + ], count: 5, - value: 'abcd', }, depth: 1, }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts index cfbcfe5a04e59..78ff0e8e1e5dd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts @@ -5,10 +5,11 @@ * 2.0. */ +import { SearchTypes } from '../../../../common/detection_engine/types'; import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema'; import { SIGNALS_TEMPLATE_VERSION } from '../routes/index/get_signals_template'; import { isEventTypeSignal } from './build_event_type_signal'; -import { Signal, Ancestor, BaseSignalHit } from './types'; +import { Signal, Ancestor, BaseSignalHit, ThresholdResult } from './types'; /** * Takes a parent signal or event document and extracts the information needed for the corresponding entry in the child @@ -95,16 +96,24 @@ export const buildSignal = (docs: BaseSignalHit[], rule: RulesSchema): Signal => }; }; +const isThresholdResult = (thresholdResult: SearchTypes): thresholdResult is ThresholdResult => { + return typeof thresholdResult === 'object'; +}; + /** * Creates signal fields that are only available in the special case where a signal has only 1 parent signal/event. * @param doc The parent signal/event of the new signal to be built. */ export const additionalSignalFields = (doc: BaseSignalHit) => { + const thresholdResult = doc._source.threshold_result; + if (thresholdResult != null && !isThresholdResult(thresholdResult)) { + throw new Error(`threshold_result failed to validate: ${thresholdResult}`); + } return { parent: buildParent(removeClashes(doc)), original_time: doc._source['@timestamp'], // This field has already been replaced with timestampOverride, if provided. original_event: doc._source.event ?? undefined, - threshold_result: doc._source.threshold_result, + threshold_result: thresholdResult, original_signal: doc._source.signal != null && !isEventTypeSignal(doc) ? doc._source.signal : undefined, }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts index 713178345361d..56d71048bb81b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts @@ -9,27 +9,41 @@ import { loggingSystemMock } from '../../../../../../../src/core/server/mocks'; import { sampleDocNoSortId, sampleDocSearchResultsNoSortId } from './__mocks__/es_results'; import { transformThresholdResultsToEcs } from './bulk_create_threshold_signals'; import { calculateThresholdSignalUuid } from './utils'; +import { Threshold } from '../../../../common/detection_engine/schemas/common/schemas'; describe('transformThresholdResultsToEcs', () => { it('should return transformed threshold results', () => { - const threshold = { - field: 'source.ip', + const threshold: Threshold = { + field: ['source.ip', 'host.name'], value: 1, + cardinality_field: 'destination.ip', + cardinality_value: 5, }; const startedAt = new Date('2020-12-17T16:27:00Z'); const transformedResults = transformThresholdResultsToEcs( { ...sampleDocSearchResultsNoSortId('abcd'), aggregations: { - threshold: { + 'threshold_0:source.ip': { buckets: [ { key: '127.0.0.1', - doc_count: 1, - top_threshold_hits: { - hits: { - hits: [sampleDocNoSortId('abcd')], - }, + doc_count: 15, + 'threshold_1:host.name': { + buckets: [ + { + key: 'garden-gnomes', + doc_count: 12, + top_threshold_hits: { + hits: { + hits: [sampleDocNoSortId('abcd')], + }, + }, + cardinality_count: { + value: 7, + }, + }, + ], }, }, ], @@ -44,7 +58,12 @@ describe('transformThresholdResultsToEcs', () => { '1234', undefined ); - const _id = calculateThresholdSignalUuid('1234', startedAt, 'source.ip', '127.0.0.1'); + const _id = calculateThresholdSignalUuid( + '1234', + startedAt, + ['source.ip', 'host.name'], + '127.0.0.1,garden-gnomes' + ); expect(transformedResults).toEqual({ took: 10, timed_out: false, @@ -67,10 +86,25 @@ describe('transformThresholdResultsToEcs', () => { _id, _index: 'test', _source: { - '@timestamp': ['2020-04-20T21:27:45+0000'], + '@timestamp': '2020-04-20T21:27:45+0000', threshold_result: { - count: 1, - value: '127.0.0.1', + terms: [ + { + field: 'source.ip', + value: '127.0.0.1', + }, + { + field: 'host.name', + value: 'garden-gnomes', + }, + ], + cardinality: [ + { + field: 'destination.ip', + value: 7, + }, + ], + count: 12, }, }, }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts index dd9e1e97a2b73..29fd189bb34f3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts @@ -18,12 +18,13 @@ import { AlertInstanceState, AlertServices, } from '../../../../../alerts/server'; -import { RuleAlertAction } from '../../../../common/detection_engine/types'; +import { BaseHit, RuleAlertAction } from '../../../../common/detection_engine/types'; import { RuleTypeParams, RefreshTypes } from '../types'; import { singleBulkCreate, SingleBulkCreateResponse } from './single_bulk_create'; -import { SignalSearchResponse, ThresholdAggregationBucket } from './types'; -import { calculateThresholdSignalUuid } from './utils'; +import { calculateThresholdSignalUuid, getThresholdAggregationParts } from './utils'; import { BuildRuleMessage } from './rule_messages'; +import { TermAggregationBucket } from '../../types'; +import { MultiAggBucket, SignalSearchResponse, SignalSource } from './types'; interface BulkCreateThresholdSignalsParams { actions: RuleAlertAction[]; @@ -73,52 +74,150 @@ const getTransformedHits = ( logger.warn(`No hits returned, but totalResults >= threshold.value (${threshold.value})`); return []; } + const timestampArray = get(timestampOverride ?? '@timestamp', hit.fields); + if (timestampArray == null) { + return []; + } + const timestamp = timestampArray[0]; + if (typeof timestamp !== 'string') { + return []; + } const source = { - '@timestamp': get(timestampOverride ?? '@timestamp', hit.fields), + '@timestamp': timestamp, threshold_result: { + terms: [ + { + value: ruleId, + }, + ], count: totalResults, - value: ruleId, }, }; return [ { _index: inputIndex, - _id: calculateThresholdSignalUuid(ruleId, startedAt, threshold.field), + _id: calculateThresholdSignalUuid( + ruleId, + startedAt, + Array.isArray(threshold.field) ? threshold.field : [threshold.field] + ), _source: source, }, ]; } - if (!results.aggregations?.threshold) { + const aggParts = results.aggregations && getThresholdAggregationParts(results.aggregations); + if (!aggParts) { return []; } - return results.aggregations.threshold.buckets - .map( - ({ key, doc_count: docCount, top_threshold_hits: topHits }: ThresholdAggregationBucket) => { - const hit = topHits.hits.hits[0]; - if (hit == null) { - return null; + const getCombinations = (buckets: TermAggregationBucket[], i: number, field: string) => { + return buckets.reduce((acc: MultiAggBucket[], bucket: TermAggregationBucket) => { + if (i < threshold.field.length - 1) { + const nextLevelIdx = i + 1; + const nextLevelAggParts = getThresholdAggregationParts(bucket, nextLevelIdx); + if (nextLevelAggParts == null) { + throw new Error('Something went horribly wrong'); } - - const source = { - '@timestamp': get(timestampOverride ?? '@timestamp', hit.fields), - threshold_result: { - count: docCount, - value: key, - }, + const nextLevelPath = `['${nextLevelAggParts.name}']['buckets']`; + const nextBuckets = get(nextLevelPath, bucket); + const combinations = getCombinations(nextBuckets, nextLevelIdx, nextLevelAggParts.field); + combinations.forEach((val) => { + const el = { + terms: [ + { + field, + value: bucket.key, + }, + ...val.terms, + ], + cardinality: val.cardinality, + topThresholdHits: val.topThresholdHits, + docCount: val.docCount, + }; + acc.push(el); + }); + } else { + const el = { + terms: [ + { + field, + value: bucket.key, + }, + ], + cardinality: !isEmpty(threshold.cardinality_field) + ? [ + { + field: Array.isArray(threshold.cardinality_field) + ? threshold.cardinality_field[0] + : threshold.cardinality_field!, + value: bucket.cardinality_count!.value, + }, + ] + : undefined, + topThresholdHits: bucket.top_threshold_hits, + docCount: bucket.doc_count, }; + acc.push(el); + } - return { - _index: inputIndex, - _id: calculateThresholdSignalUuid(ruleId, startedAt, threshold.field, key), - _source: source, - }; + return acc; + }, []); + }; + + return getCombinations(results.aggregations[aggParts.name].buckets, 0, aggParts.field).reduce( + (acc: Array>, bucket) => { + const hit = bucket.topThresholdHits?.hits.hits[0]; + if (hit == null) { + return acc; } - ) - .filter((bucket: ThresholdAggregationBucket) => bucket != null); + + const timestampArray = get(timestampOverride ?? '@timestamp', hit.fields); + if (timestampArray == null) { + return acc; + } + + const timestamp = timestampArray[0]; + if (typeof timestamp !== 'string') { + return acc; + } + + const source = { + '@timestamp': timestamp, + threshold_result: { + terms: bucket.terms.map((term) => { + return { + field: term.field, + value: term.value, + }; + }), + cardinality: bucket.cardinality?.map((cardinality) => { + return { + field: cardinality.field, + value: cardinality.value, + }; + }), + count: bucket.docCount, + }, + }; + + acc.push({ + _index: inputIndex, + _id: calculateThresholdSignalUuid( + ruleId, + startedAt, + Array.isArray(threshold.field) ? threshold.field : [threshold.field], + bucket.terms.map((term) => term.value).join(',') + ), + _source: source, + }); + + return acc; + }, + [] + ); }; export const transformThresholdResultsToEcs = ( @@ -149,7 +248,7 @@ export const transformThresholdResultsToEcs = ( }, }; - delete thresholdResults.aggregations; // no longer needed + delete thresholdResults.aggregations; // delete because no longer needed set(thresholdResults, 'results.hits.total', transformedHits.length); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts index 6144f1f4b3823..7796346e9876d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { set } from '@elastic/safer-lodash-set'; import { isEmpty } from 'lodash/fp'; import { @@ -49,38 +50,68 @@ export const findThresholdSignals = async ({ searchDuration: string; searchErrors: string[]; }> => { + const thresholdFields = Array.isArray(threshold.field) ? threshold.field : [threshold.field]; + const aggregations = threshold && !isEmpty(threshold.field) - ? { - threshold: { + ? thresholdFields.reduce((acc, field, i) => { + const aggPath = [...Array(i + 1).keys()] + .map((j) => { + return `['threshold_${j}:${thresholdFields[j]}']`; + }) + .join(`['aggs']`); + set(acc, aggPath, { terms: { - field: threshold.field, - min_doc_count: threshold.value, + field, + min_doc_count: threshold.value, // not needed on parent agg, but can help narrow down result set size: 10000, // max 10k buckets }, - aggs: { - // Get the most recent hit per bucket - top_threshold_hits: { - top_hits: { - sort: [ - { - [timestampOverride ?? '@timestamp']: { - order: 'desc', - }, + }); + if (i === threshold.field.length - 1) { + const topHitsAgg = { + top_hits: { + sort: [ + { + [timestampOverride ?? '@timestamp']: { + order: 'desc', }, - ], - fields: [ - { - field: '*', - include_unmapped: true, + }, + ], + fields: [ + { + field: '*', + include_unmapped: true, + }, + ], + size: 1, + }, + }; + // TODO: support case where threshold fields are not supplied, but cardinality is? + if (!isEmpty(threshold.cardinality_field)) { + set(acc, `${aggPath}['aggs']`, { + top_threshold_hits: topHitsAgg, + cardinality_count: { + cardinality: { + field: threshold.cardinality_field, + }, + }, + cardinality_check: { + bucket_selector: { + buckets_path: { + cardinalityCount: 'cardinality_count', }, - ], - size: 1, + script: `params.cardinalityCount >= ${threshold.cardinality_value}`, // TODO: cardinality operator + }, }, - }, - }, - }, - } + }); + } else { + set(acc, `${aggPath}['aggs']`, { + top_threshold_hits: topHitsAgg, + }); + } + } + return acc; + }, {}) : {}; return singleSearchAfter({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts index da7ee8796afbf..710a925fe315b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts @@ -40,7 +40,12 @@ export const signalSchema = schema.object({ severityMapping: schema.nullable(schema.arrayOf(schema.object({}, { unknowns: 'allow' }))), threat: schema.nullable(schema.arrayOf(schema.object({}, { unknowns: 'allow' }))), threshold: schema.maybe( - schema.object({ field: schema.nullable(schema.string()), value: schema.number() }) + schema.object({ + field: schema.nullable(schema.oneOf([schema.string(), schema.arrayOf(schema.string())])), + value: schema.number(), + cardinality_field: schema.nullable(schema.string()), // TODO: depends on `field` being defined? + cardinality_value: schema.nullable(schema.number()), + }) ), timestampOverride: schema.nullable(schema.string()), to: schema.string(), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index ecb36a8b050d9..98c9dd41d179c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -374,6 +374,10 @@ export const signalRulesAlertType = ({ } else if (isThresholdRule(type) && threshold) { const inputIndex = await getInputIndex(services, version, index); + const thresholdFields = Array.isArray(threshold.field) + ? threshold.field + : [threshold.field]; + const { filters: bucketFilters, searchErrors: previousSearchErrors, @@ -384,7 +388,7 @@ export const signalRulesAlertType = ({ services, logger, ruleId, - bucketByField: threshold.field, + bucketByFields: thresholdFields, timestampOverride, buildRuleMessage, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_find_previous_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_find_previous_signals.ts index dbad1d12d2be6..8ed5929e72504 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_find_previous_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_find_previous_signals.ts @@ -24,7 +24,7 @@ interface FindPreviousThresholdSignalsParams { services: AlertServices; logger: Logger; ruleId: string; - bucketByField: string; + bucketByFields: string[]; timestampOverride: TimestampOverrideOrUndefined; buildRuleMessage: BuildRuleMessage; } @@ -36,7 +36,7 @@ export const findPreviousThresholdSignals = async ({ services, logger, ruleId, - bucketByField, + bucketByFields, timestampOverride, buildRuleMessage, }: FindPreviousThresholdSignalsParams): Promise<{ @@ -44,22 +44,6 @@ export const findPreviousThresholdSignals = async ({ searchDuration: string; searchErrors: string[]; }> => { - const aggregations = { - threshold: { - terms: { - field: 'signal.threshold_result.value', - size: 10000, - }, - aggs: { - lastSignalTimestamp: { - max: { - field: 'signal.original_time', // timestamp of last event captured by bucket - }, - }, - }, - }, - }; - const filter = { bool: { must: [ @@ -68,17 +52,18 @@ export const findPreviousThresholdSignals = async ({ 'signal.rule.rule_id': ruleId, }, }, - { - term: { - 'signal.rule.threshold.field': bucketByField, - }, - }, + ...bucketByFields.map((field) => { + return { + term: { + 'signal.rule.threshold.field': field, + }, + }; + }), ], }, }; return singleSearchAfter({ - aggregations, searchAfterSortId: undefined, timestampOverride, index: indexPattern, @@ -87,7 +72,7 @@ export const findPreviousThresholdSignals = async ({ services, logger, filter, - pageSize: 0, + pageSize: 10000, // TODO: multiple pages? buildRuleMessage, excludeDocsWithTimestampOverride: false, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.test.ts new file mode 100644 index 0000000000000..ed9aa9a5ba698 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.test.ts @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { alertsMock, AlertServicesMock } from '../../../../../alerts/server/mocks'; +import { mockLogger, sampleWrappedThresholdSignalHit } from './__mocks__/es_results'; +import { getThresholdBucketFilters } from './threshold_get_bucket_filters'; +import { buildRuleMessageFactory } from './rule_messages'; + +const buildRuleMessage = buildRuleMessageFactory({ + id: 'fake id', + ruleId: 'fake rule id', + index: 'fakeindex', + name: 'fake name', +}); + +describe('thresholdGetBucketFilters', () => { + let mockService: AlertServicesMock; + + beforeEach(() => { + jest.clearAllMocks(); + mockService = alertsMock.createAlertServices(); + }); + + it('should generate filters for threshold signal detection with dupe mitigation', async () => { + mockService.callCluster.mockResolvedValue({ + took: 10, + timed_out: false, + _shards: { + total: 10, + successful: 10, + failed: 0, + skipped: 0, + }, + hits: { + total: 1, + max_score: 100, + hits: [sampleWrappedThresholdSignalHit()], + }, + }); + const result = await getThresholdBucketFilters({ + from: 'now-6m', + to: 'now', + indexPattern: ['*'], + services: mockService, + logger: mockLogger, + ruleId: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9', + bucketByFields: ['host.name'], + timestampOverride: undefined, + buildRuleMessage, + }); + expect(result).toEqual({ + filters: [ + { + bool: { + must_not: [ + { + bool: { + filter: [ + { + range: { + '@timestamp': { + lte: '2021-02-16T17:37:34.275Z', + }, + }, + }, + { + term: { + 'host.name': 'a hostname', + }, + }, + ], + }, + }, + ], + }, + }, + ], + searchErrors: [], + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.ts index 96224be181f47..e1727c0361afc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.ts @@ -5,11 +5,13 @@ * 2.0. */ +import crypto from 'crypto'; import { isEmpty } from 'lodash'; import { Filter } from 'src/plugins/data/common'; import { ESFilter } from '../../../../../../typings/elasticsearch'; +import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema'; import { TimestampOverrideOrUndefined } from '../../../../common/detection_engine/schemas/common/schemas'; import { AlertInstanceContext, @@ -17,7 +19,7 @@ import { AlertServices, } from '../../../../../alerts/server'; import { Logger } from '../../../../../../../src/core/server'; -import { ThresholdQueryBucket } from './types'; +import { ThresholdSignalHistory, ThresholdSignalHistoryRecord } from './types'; import { BuildRuleMessage } from './rule_messages'; import { findPreviousThresholdSignals } from './threshold_find_previous_signals'; @@ -28,7 +30,7 @@ interface GetThresholdBucketFiltersParams { services: AlertServices; logger: Logger; ruleId: string; - bucketByField: string; + bucketByFields: string[]; timestampOverride: TimestampOverrideOrUndefined; buildRuleMessage: BuildRuleMessage; } @@ -40,7 +42,7 @@ export const getThresholdBucketFilters = async ({ services, logger, ruleId, - bucketByField, + bucketByFields, timestampOverride, buildRuleMessage, }: GetThresholdBucketFiltersParams): Promise<{ @@ -54,20 +56,85 @@ export const getThresholdBucketFilters = async ({ services, logger, ruleId, - bucketByField, + bucketByFields, timestampOverride, buildRuleMessage, }); - const filters = searchResult.aggregations.threshold.buckets.reduce( - (acc: ESFilter[], bucket: ThresholdQueryBucket): ESFilter[] => { + const thresholdSignalHistory = searchResult.hits.hits.reduce( + (acc, hit) => { + if (!hit._source) { + return acc; + } + + const terms = bucketByFields.map((field) => { + let signalTerms = hit._source.signal?.threshold_result?.terms; + + // Handle pre-7.12 signals + if (signalTerms == null) { + signalTerms = [ + { + field: (((hit._source.rule as RulesSchema).threshold as unknown) as { field: string }) + .field, + value: ((hit._source.signal?.threshold_result as unknown) as { value: string }).value, + }, + ]; + } else if (isEmpty(signalTerms)) { + signalTerms = []; + } + + const result = signalTerms.filter((resultField) => { + return resultField.field === field; + }); + + return { + field, + value: result[0].value, + }; + }); + + const hash = crypto + .createHash('sha256') + .update( + terms + .sort((term1, term2) => (term1.field > term2.field ? 1 : -1)) + .map((field) => { + return field.value; + }) + .join(',') + ) + .digest('hex'); + + const existing = acc[hash]; + const originalTime = + hit._source.signal?.original_time != null + ? new Date(hit._source.signal?.original_time).getTime() + : undefined; + + if (existing != null) { + if (originalTime && originalTime > existing.lastSignalTimestamp) { + acc[hash].lastSignalTimestamp = originalTime; + } + } else if (originalTime) { + acc[hash] = { + terms, + lastSignalTimestamp: originalTime, + }; + } + return acc; + }, + {} + ); + + const filters = Object.values(thresholdSignalHistory).reduce( + (acc: ESFilter[], bucket: ThresholdSignalHistoryRecord): ESFilter[] => { const filter = { bool: { filter: [ { range: { [timestampOverride ?? '@timestamp']: { - lte: bucket.lastSignalTimestamp.value_as_string, + lte: new Date(bucket.lastSignalTimestamp).toISOString(), }, }, }, @@ -75,11 +142,15 @@ export const getThresholdBucketFilters = async ({ }, } as ESFilter; - if (!isEmpty(bucketByField)) { - (filter.bool.filter as ESFilter[]).push({ - term: { - [bucketByField]: bucket.key, - }, + if (!isEmpty(bucketByFields)) { + bucket.terms.forEach((term) => { + if (term.field != null) { + (filter.bool.filter as ESFilter[]).push({ + term: { + [term.field]: `${term.value}`, + }, + }); + } }); } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index f7ac0425b2f2e..e5ca1f6a60456 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -17,7 +17,7 @@ import { AlertExecutorOptions, AlertServices, } from '../../../../../alerts/server'; -import { BaseSearchResponse, SearchResponse, TermAggregationBucket } from '../../types'; +import { BaseSearchResponse, SearchHit, SearchResponse, TermAggregationBucket } from '../../types'; import { EqlSearchResponse, BaseHit, @@ -50,8 +50,27 @@ export interface SignalsStatusParams { } export interface ThresholdResult { + terms?: Array<{ + field?: string; + value: string; + }>; + cardinality?: Array<{ + field: string; + value: number; + }>; count: number; - value: string; +} + +export interface ThresholdSignalHistoryRecord { + terms: Array<{ + field?: string; + value: SearchTypes; + }>; + lastSignalTimestamp: number; +} + +export interface ThresholdSignalHistory { + [hash: string]: ThresholdSignalHistoryRecord; } export interface SignalSource { @@ -74,8 +93,9 @@ export interface SignalSource { }; // signal.depth doesn't exist on pre-7.10 signals depth?: number; + original_time?: string; + threshold_result?: ThresholdResult; }; - threshold_result?: ThresholdResult; } export interface BulkItem { @@ -276,6 +296,28 @@ export interface SearchAfterAndBulkCreateReturnType { export interface ThresholdAggregationBucket extends TermAggregationBucket { top_threshold_hits: BaseSearchResponse; + cardinality_count: { + value: number; + }; +} + +export interface MultiAggBucket { + cardinality?: Array<{ + field: string; + value: number; + }>; + terms: Array<{ + field: string; + value: string; + }>; + docCount: number; + topThresholdHits?: + | { + hits: { + hits: SearchHit[]; + }; + } + | undefined; } export interface ThresholdQueryBucket extends TermAggregationBucket { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts index f7e1eb7622779..7888bb6deaab7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts @@ -1445,13 +1445,13 @@ describe('utils', () => { describe('calculateThresholdSignalUuid', () => { it('should generate a uuid without key', () => { const startedAt = new Date('2020-12-17T16:27:00Z'); - const signalUuid = calculateThresholdSignalUuid('abcd', startedAt, 'agent.name'); + const signalUuid = calculateThresholdSignalUuid('abcd', startedAt, ['agent.name']); expect(signalUuid).toEqual('a4832768-a379-583a-b1a2-e2ce2ad9e6e9'); }); it('should generate a uuid with key', () => { const startedAt = new Date('2019-11-18T13:32:00Z'); - const signalUuid = calculateThresholdSignalUuid('abcd', startedAt, 'host.ip', '1.2.3.4'); + const signalUuid = calculateThresholdSignalUuid('abcd', startedAt, ['host.ip'], '1.2.3.4'); expect(signalUuid).toEqual('ee8870dc-45ff-5e6c-a2f9-80886651ce03'); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts index 323986e6ffecb..58bf22be97bf8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts @@ -855,7 +855,7 @@ export const createTotalHitsFromSearchResult = ({ export const calculateThresholdSignalUuid = ( ruleId: string, startedAt: Date, - thresholdField: string, + thresholdFields: string[], key?: string ): string => { // used to generate constant Threshold Signals ID when run with the same params @@ -863,7 +863,31 @@ export const calculateThresholdSignalUuid = ( const startedAtString = startedAt.toISOString(); const keyString = key ?? ''; - const baseString = `${ruleId}${startedAtString}${thresholdField}${keyString}`; + const baseString = `${ruleId}${startedAtString}${thresholdFields.join(',')}${keyString}`; return uuidv5(baseString, NAMESPACE_ID); }; + +export const getThresholdAggregationParts = ( + data: object, + index?: number +): + | { + field: string; + index: number; + name: string; + } + | undefined => { + const idx = index != null ? index.toString() : '\\d'; + const pattern = `threshold_(?${idx}):(?.*)`; + for (const key of Object.keys(data)) { + const matches = key.match(pattern); + if (matches != null && matches.groups?.name != null && matches.groups?.index != null) { + return { + field: matches.groups.name, + index: parseInt(matches.groups.index, 10), + name: key, + }; + } + } +}; diff --git a/x-pack/plugins/security_solution/server/lib/types.ts b/x-pack/plugins/security_solution/server/lib/types.ts index 9d030e0d59bc6..a8616dc1c57d1 100644 --- a/x-pack/plugins/security_solution/server/lib/types.ts +++ b/x-pack/plugins/security_solution/server/lib/types.ts @@ -75,8 +75,8 @@ export interface SearchHits { max_score: number; hits: Array< BaseHit & { - _type: string; - _score: number; + _type?: string; + _score?: number; _version?: number; _explanation?: Explanation; // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -107,6 +107,14 @@ export type SearchHit = SearchResponse['hits']['hits'][0]; export interface TermAggregationBucket { key: string; doc_count: number; + top_threshold_hits?: { + hits: { + hits: SearchHit[]; + }; + }; + cardinality_count?: { + value: number; + }; } export interface TermAggregation { diff --git a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/constants.ts b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/constants.ts index 5ed324496e609..15d0e2d5494b8 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/constants.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/constants.ts @@ -24,6 +24,7 @@ export const TIMELINE_EVENTS_FIELDS = [ 'signal.rule.version', 'signal.rule.severity', 'signal.rule.risk_score', + 'signal.threshold_result', 'event.code', 'event.module', 'event.action', diff --git a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.test.ts b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.test.ts index 5c8f7f87a6c49..10bb606dc2387 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.test.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.test.ts @@ -320,6 +320,7 @@ describe('#formatTimelineData', () => { signal: { original_time: ['2021-01-09T13:39:32.595Z'], status: ['open'], + threshold_result: ['{"count":10000,"value":"2a990c11-f61b-4c8e-b210-da2574e9f9db"}'], rule: { building_block_type: [], exceptions_list: [],