Skip to content

Commit

Permalink
Manual rule run tests (#187958)
Browse files Browse the repository at this point in the history 
## FTR tests for manual rule run:

For all rule types we cover
- that manual rule run can generate alerts
- that it not create duplicates (except case for threshold and esql)
- that suppression work per execution (except trhreshold)
- that suppression work per time period

For IM rule also covered that `threat_query `not affected by manual rule
run range

Also covered several common cases, but tests are created only for custom
query rule:

- disabling rule, after manual rule run execution started, not affecting
manual run executions
- changing name of the rule after manual rule run started, not affecting
alert generated by manual rule run executions


related:
elastic/security-team#9826 (comment)

---------

Co-authored-by: Elastic Machine <[email protected]>
  • Loading branch information
@nkhristinin @elasticmachine
nkhristinin and elasticmachine authored Jul 19, 2024
1 parent 3dd2034 commit 6aaccd6
Show file tree
Hide file tree
Showing 16 changed files with 1,997 additions and 45 deletions.
1 change: 1 addition & 0 deletions .github/CODEOWNERS
View file
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
Expand Up @@ -1602,6 +1602,7 @@ x-pack/test/security_solution_cypress/cypress/tasks/expandable_flyout @elastic/
/x-pack/test/security_solution_cypress/cypress/e2e/detection_response/detection_engine @elastic/security-detection-engine

/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine @elastic/security-detection-engine
/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/rules/rule_gaps.ts @elastic/security-detection-engine
/x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists @elastic/security-detection-engine

## Security Threat Intelligence - Under Security Platform
Expand Down
1 change: 1 addition & 0 deletions ...tion_engine/rule_execution_logic/trial_license_complete_tier/configs/serverless.config.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ export default createTestConfig({
'bulkCustomHighlightedFieldsEnabled',
'alertSuppressionForMachineLearningRuleEnabled',
'alertSuppressionForEsqlRuleEnabled',
'manualRuleRunEnabled',
])}`,
],
});
335 changes: 333 additions & 2 deletions ...n_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/custom_query.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import {
import { flattenWithPrefix } from '@kbn/securitysolution-rules';
import { Rule } from '@kbn/alerting-plugin/common';
import { BaseRuleParams } from '@kbn/security-solution-plugin/server/lib/detection_engine/rule_schema';

import moment from 'moment';
import { orderBy } from 'lodash';
import { v4 as uuidv4 } from 'uuid';

Expand Down Expand Up @@ -60,6 +60,9 @@ import {
createRuleThroughAlertingEndpoint,
getRuleSavedObjectWithLegacyInvestigationFields,
dataGeneratorFactory,
scheduleRuleRun,
stopAllManualRuns,
waitForBackfillExecuted,
} from '../../../../utils';
import {
createRule,
Expand Down Expand Up @@ -867,7 +870,7 @@ export default ({ getService }: FtrProviderContext) => {
};
const createdRule = await createRule(supertest, log, rule);
const alerts = await getAlerts(supertest, log, es, createdRule);
expect(alerts.hits.hits.length).toEqual(1);
expect(alerts.hits.hits).toHaveLength(1);
expect(alerts.hits.hits[0]._source).toEqual({
...alerts.hits.hits[0]._source,
[ALERT_SUPPRESSION_TERMS]: [
Expand Down Expand Up @@ -2419,5 +2422,333 @@ export default ({ getService }: FtrProviderContext) => {
expect(alertsAfterEnable.hits.hits.length > 0).toEqual(true);
});
});

// skipped on MKI since feature flags are not supported there
describe('@skipInServerlessMKI manual rule run', () => {
const { indexListOfDocuments } = dataGeneratorFactory({
es,
index: 'ecs_compliant',
log,
});

beforeEach(async () => {
await stopAllManualRuns(supertest);
await esArchiver.load('x-pack/test/functional/es_archives/security_solution/ecs_compliant');
});

afterEach(async () => {
await stopAllManualRuns(supertest);
await esArchiver.unload(
'x-pack/test/functional/es_archives/security_solution/ecs_compliant'
);
});

it('alerts when run on a time range that the rule has not previously seen, and deduplicates if run there more than once', async () => {
const id = uuidv4();
const firstTimestamp = moment(new Date()).subtract(3, 'h').toISOString();
const secondTimestamp = new Date().toISOString();
const firstDocument = {
id,
'@timestamp': firstTimestamp,
agent: {
name: 'agent-1',
},
};
const secondDocument = {
id,
'@timestamp': secondTimestamp,
agent: {
name: 'agent-2',
},
};
await indexListOfDocuments([firstDocument, secondDocument]);

const rule: QueryRuleCreateProps = {
...getRuleForAlertTesting(['ecs_compliant']),
rule_id: 'rule-1',
query: `id:${id}`,
from: 'now-1h',
interval: '1h',
};
const createdRule = await createRule(supertest, log, rule);
const alerts = await getAlerts(supertest, log, es, createdRule);

expect(alerts.hits.hits).toHaveLength(1);

const backfill = await scheduleRuleRun(supertest, [createdRule.id], {
startDate: moment(firstTimestamp).subtract(5, 'm'),
endDate: moment(firstTimestamp).add(5, 'm'),
});

await waitForBackfillExecuted(backfill, [createdRule.id], { supertest, log });
const allNewAlerts = await getAlerts(supertest, log, es, createdRule);
expect(allNewAlerts.hits.hits).toHaveLength(2);

const secondBackfill = await scheduleRuleRun(supertest, [createdRule.id], {
startDate: moment(firstTimestamp).subtract(5, 'm'),
endDate: moment(firstTimestamp).add(5, 'm'),
});

await waitForBackfillExecuted(secondBackfill, [createdRule.id], { supertest, log });
const allNewAlertsAfter2ManualRuns = await getAlerts(supertest, log, es, createdRule);
expect(allNewAlertsAfter2ManualRuns.hits.hits.length).toEqual(2);
});

it('does not alert if the manual run overlaps with a previous scheduled rule execution', async () => {
const id = uuidv4();
const firstTimestamp = moment(new Date()).subtract(5, 'm').toISOString();
const firstDocument = {
id,
'@timestamp': firstTimestamp,
agent: {
name: 'agent-1',
},
};
await indexListOfDocuments([firstDocument]);

const rule: QueryRuleCreateProps = {
...getRuleForAlertTesting(['ecs_compliant']),
rule_id: 'rule-1',
query: `id:${id}`,
from: 'now-1h',
interval: '1h',
};
const createdRule = await createRule(supertest, log, rule);
const alerts = await getAlerts(supertest, log, es, createdRule);

expect(alerts.hits.hits).toHaveLength(1);

const backfill = await scheduleRuleRun(supertest, [createdRule.id], {
startDate: moment(firstTimestamp).subtract(5, 'm'),
endDate: moment().subtract(1, 'm'),
});

await waitForBackfillExecuted(backfill, [createdRule.id], { supertest, log });
const allNewAlerts = await getAlerts(supertest, log, es, createdRule);
expect(allNewAlerts.hits.hits).toHaveLength(1);
});

it('manual rule runs should not be affected if the rule is disabled after manual rule run execution', async () => {
const id = uuidv4();
const firstTimestamp = moment(new Date()).subtract(4, 'h').toISOString();
const secondTimestamp = moment(new Date()).subtract(3, 'h');
const firstDocument = {
id,
'@timestamp': firstTimestamp,
agent: {
name: 'agent-1',
},
};
const secondDocument = {
id,
'@timestamp': secondTimestamp,
agent: {
name: 'agent-2',
},
};

await indexListOfDocuments([firstDocument, secondDocument]);

const rule: QueryRuleCreateProps = {
...getRuleForAlertTesting(['ecs_compliant']),
rule_id: 'rule-1',
query: `id:${id}`,
from: 'now-1h',
interval: '1h',
};
const createdRule = await createRule(supertest, log, rule);
const alerts = await getAlerts(supertest, log, es, createdRule);

expect(alerts.hits.hits).toHaveLength(0);

const backfill = await scheduleRuleRun(supertest, [createdRule.id], {
startDate: moment(firstTimestamp).subtract(10, 'h'),
endDate: moment().subtract(1, 'm'),
});

await patchRule(supertest, log, { id: createdRule.id, enabled: false });

await waitForBackfillExecuted(backfill, [createdRule.id], { supertest, log });

const allNewAlerts = await getAlerts(supertest, log, es, createdRule);
expect(allNewAlerts.hits.hits).toHaveLength(2);
});

it("change rule params after manual rule run starts, shoulnd't affect fields of alerts generated by manual run", async () => {
const id = uuidv4();
const firstTimestamp = moment(new Date()).subtract(4, 'h').toISOString();
const secondTimestamp = moment(new Date()).subtract(3, 'h');
const firstDocument = {
id,
'@timestamp': firstTimestamp,
agent: {
name: 'agent-1',
},
};
const secondDocument = {
id,
'@timestamp': secondTimestamp,
agent: {
name: 'agent-2',
},
};

await indexListOfDocuments([firstDocument, secondDocument]);

const rule: QueryRuleCreateProps = {
...getRuleForAlertTesting(['ecs_compliant']),
name: 'original rule name',
rule_id: 'rule-1',
query: `id:${id}`,
from: 'now-1h',
interval: '1h',
};
const createdRule = await createRule(supertest, log, rule);
const alerts = await getAlerts(supertest, log, es, createdRule);

expect(alerts.hits.hits).toHaveLength(0);

const backfill = await scheduleRuleRun(supertest, [createdRule.id], {
startDate: moment(firstTimestamp).subtract(10, 'h'),
endDate: moment().subtract(1, 'm'),
});

await patchRule(supertest, log, { id: createdRule.id, name: 'new rule name' });

await waitForBackfillExecuted(backfill, [createdRule.id], { supertest, log });

const allNewAlerts = await getAlerts(supertest, log, es, createdRule);
expect(allNewAlerts.hits.hits).toHaveLength(2);
expect(allNewAlerts.hits.hits[0]?._source?.['kibana.alert.rule.name']).toEqual(
'original rule name'
);
expect(allNewAlerts.hits.hits[1]?._source?.['kibana.alert.rule.name']).toEqual(
'original rule name'
);
});

it('supression per rule execution should work for manual rule runs', async () => {
const id = uuidv4();
const firstTimestamp = moment(new Date()).subtract(3, 'h');
const firstDocument = {
id,
'@timestamp': firstTimestamp.toISOString(),
agent: {
name: 'agent-1',
},
};
const secondDocument = {
id,
'@timestamp': moment(firstTimestamp).add(1, 'm').toISOString(),
agent: {
name: 'agent-1',
},
};
const thirdDocument = {
id,
'@timestamp': moment(firstTimestamp).add(3, 'm').toISOString(),
agent: {
name: 'agent-1',
},
};

await indexListOfDocuments([firstDocument, secondDocument, thirdDocument]);

const rule: QueryRuleCreateProps = {
...getRuleForAlertTesting(['ecs_compliant']),
rule_id: 'rule-1',
query: `id:${id}`,
from: 'now-1h',
interval: '1h',
alert_suppression: {
group_by: ['agent.name'],
},
};
const createdRule = await createRule(supertest, log, rule);
const alerts = await getAlerts(supertest, log, es, createdRule);

expect(alerts.hits.hits).toHaveLength(0);

const backfill = await scheduleRuleRun(supertest, [createdRule.id], {
startDate: moment(firstTimestamp).subtract(5, 'm'),
endDate: moment(firstTimestamp).add(10, 'm'),
});

await waitForBackfillExecuted(backfill, [createdRule.id], { supertest, log });
const allNewAlerts = await getAlerts(supertest, log, es, createdRule);
expect(allNewAlerts.hits.hits).toHaveLength(1);
});

it('supression with time window should work for manual rule runs and update alert', async () => {
const id = uuidv4();
const firstTimestamp = moment(new Date()).subtract(3, 'h');
const firstDocument = {
id,
'@timestamp': firstTimestamp.toISOString(),
agent: {
name: 'agent-1',
},
};

await indexListOfDocuments([firstDocument]);
const rule: QueryRuleCreateProps = {
...getRuleForAlertTesting(['ecs_compliant']),
rule_id: 'rule-1',
query: `id:${id}`,
from: 'now-3h',
interval: '30m',
alert_suppression: {
group_by: ['agent.name'],
duration: {
value: 500,
unit: 'm',
},
},
};

const createdRule = await createRule(supertest, log, rule);
const alerts = await getAlerts(supertest, log, es, createdRule);

expect(alerts.hits.hits).toHaveLength(0);

// generate alert in the past
const backfill = await scheduleRuleRun(supertest, [createdRule.id], {
startDate: moment(firstTimestamp).subtract(5, 'm'),
endDate: moment(firstTimestamp).add(5, 'm'),
});

await waitForBackfillExecuted(backfill, [createdRule.id], { supertest, log });
const allNewAlerts = await getAlerts(supertest, log, es, createdRule);
expect(allNewAlerts.hits.hits).toHaveLength(1);

// now we will ingest new event, and manual rule run should update original alert
const secondDocument = {
id,
'@timestamp': moment(firstTimestamp).add(5, 'm').toISOString(),
agent: {
name: 'agent-1',
},
};

await indexListOfDocuments([secondDocument]);

const secondBackfill = await scheduleRuleRun(supertest, [createdRule.id], {
startDate: moment(firstTimestamp).add(1, 'm'),
endDate: moment(firstTimestamp).add(120, 'm'),
});

await waitForBackfillExecuted(secondBackfill, [createdRule.id], { supertest, log });
const updatedAlerts = await getAlerts(supertest, log, es, createdRule);
expect(updatedAlerts.hits.hits).toHaveLength(1);

expect(updatedAlerts.hits.hits).toHaveLength(1);
// ALERT_SUPPRESSION_DOCS_COUNT is expected to be 1,
// but because we have serveral manual rule executions it count it incorrectly
expect(updatedAlerts.hits.hits[0]._source).toEqual({
...updatedAlerts.hits.hits[0]._source,
[ALERT_SUPPRESSION_DOCS_COUNT]: 2,
});
});
});
});
};
Loading

0 comments on commit 6aaccd6

Please sign in to comment.