Skip to content

Commit

Permalink
[Security Solution][Alerts] Return tuple of included and excluded doc…
Browse files Browse the repository at this point in the history
…s from value list exception evaluation (#130044)

* Return tuple of included and excluded docs from value list exception evaluation

* Types and tests

* Add doc comment
  • Loading branch information
marshallmain authored May 2, 2022
1 parent 848e020 commit 5f290fc
Show file tree
Hide file tree
Showing 17 changed files with 260 additions and 275 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -910,6 +910,32 @@ export const sampleDocSearchResultsNoSortIdNoHits = (
},
});

/**
*
* @param count Total number of hits to create
* @param guids List of _id values for the hits. If this array is smaller than count, the remaining hits will receive a default value.
* @param ips List of source.ip values for the hits. If this array is smaller than count, the remaining hits will receive a default value.
* @param destIps List of destination.ip values for the hits. If this array is smaller than count, the remaining hits will receive a default value.
* @param sortIds List of sort IDs. The same list is inserted into every hit.
* @returns Array of mock hits
*/
export const repeatedHitsWithSortId = (
count: number,
guids: string[],
ips?: Array<string | string[]>,
destIps?: Array<string | string[]>,
sortIds?: string[]
): SignalSourceHit[] => {
return Array.from({ length: count }).map((x, index) => ({
...sampleDocWithSortId(
guids[index],
sortIds,
ips ? ips[index] : '127.0.0.1',
destIps ? destIps[index] : '127.0.0.1'
),
}));
};

export const repeatedSearchResultsWithSortId = (
total: number,
pageSize: number,
Expand All @@ -929,14 +955,7 @@ export const repeatedSearchResultsWithSortId = (
hits: {
total,
max_score: 100,
hits: Array.from({ length: pageSize }).map((x, index) => ({
...sampleDocWithSortId(
guids[index],
sortIds,
ips ? ips[index] : '127.0.0.1',
destIps ? destIps[index] : '127.0.0.1'
),
})),
hits: repeatedHitsWithSortId(pageSize, guids, ips, destIps, sortIds),
},
});

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,15 @@ import {
RuleExecutorServices,
} from '@kbn/alerting-plugin/server';
import { GenericBulkCreateResponse } from '../rule_types/factories';
import { AnomalyResults, Anomaly } from '../../machine_learning';
import { Anomaly } from '../../machine_learning';
import { BuildRuleMessage } from './rule_messages';
import { BulkCreate, WrapHits } from './types';
import { CompleteRule, MachineLearningRuleParams } from '../schemas/rule_schemas';
import { buildReasonMessageForMlAlert } from './reason_formatters';
import { BaseFieldsLatest } from '../../../../common/detection_engine/schemas/alerts';

interface BulkCreateMlSignalsParams {
someResult: AnomalyResults;
anomalyHits: Array<estypes.SearchHit<Anomaly>>;
completeRule: CompleteRule<MachineLearningRuleParams>;
services: RuleExecutorServices<AlertInstanceState, AlertInstanceContext, 'default'>;
logger: Logger;
Expand Down Expand Up @@ -65,32 +65,23 @@ export const transformAnomalyFieldsToEcs = (anomaly: Anomaly): EcsAnomaly => {
};

const transformAnomalyResultsToEcs = (
results: AnomalyResults
): estypes.SearchResponse<EcsAnomaly> => {
const transformedHits = results.hits.hits.map(({ _source, ...rest }) => ({
results: Array<estypes.SearchHit<Anomaly>>
): Array<estypes.SearchHit<EcsAnomaly>> => {
return results.map(({ _source, ...rest }) => ({
...rest,
_source: transformAnomalyFieldsToEcs(
// @ts-expect-error @elastic/elasticsearch _source is optional
_source
),
}));

// @ts-expect-error Anomaly is not assignable to EcsAnomaly
return {
...results,
hits: {
...results.hits,
hits: transformedHits,
},
};
};

export const bulkCreateMlSignals = async (
params: BulkCreateMlSignalsParams
): Promise<GenericBulkCreateResponse<BaseFieldsLatest>> => {
const anomalyResults = params.someResult;
const anomalyResults = params.anomalyHits;
const ecsResults = transformAnomalyResultsToEcs(anomalyResults);

const wrappedDocs = params.wrapHits(ecsResults.hits.hits, buildReasonMessageForMlAlert);
const wrappedDocs = params.wrapHits(ecsResults, buildReasonMessageForMlAlert);
return params.bulkCreate(wrappedDocs);
};
Original file line number Diff line number Diff line change
Expand Up @@ -97,21 +97,21 @@ export const mlExecutor = async ({
exceptionItems,
});

const filteredAnomalyResults = await filterEventsAgainstList({
const [filteredAnomalyHits, _] = await filterEventsAgainstList({
listClient,
exceptionsList: exceptionItems,
logger,
eventSearchResult: anomalyResults,
events: anomalyResults.hits.hits,
buildRuleMessage,
});

const anomalyCount = filteredAnomalyResults.hits.hits.length;
const anomalyCount = filteredAnomalyHits.length;
if (anomalyCount) {
logger.debug(buildRuleMessage(`Found ${anomalyCount} signals from ML anomalies.`));
}
const { success, errors, bulkCreateDuration, createdItemsCount, createdItems } =
await bulkCreateMlSignals({
someResult: filteredAnomalyResults,
anomalyHits: filteredAnomalyHits,
completeRule,
services,
logger,
Expand All @@ -124,7 +124,7 @@ export const mlExecutor = async ({
// The legacy ES client does not define failures when it can be present on the structure, hence why I have the & { failures: [] }
const shardFailures =
(
filteredAnomalyResults._shards as typeof filteredAnomalyResults._shards & {
anomalyResults._shards as typeof anomalyResults._shards & {
failures: [];
}
).failures ?? [];
Expand All @@ -134,7 +134,7 @@ export const mlExecutor = async ({
return mergeReturns([
result,
createSearchAfterReturnType({
success: success && filteredAnomalyResults._shards.failed === 0,
success: success && anomalyResults._shards.failed === 0,
errors: [...errors, ...searchErrors],
createdSignalsCount: createdItemsCount,
createdSignals: createdItems,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,10 @@ import { sampleDocWithSortId } from '../__mocks__/es_results';

import { listMock } from '@kbn/lists-plugin/server/mocks';
import { getSearchListItemResponseMock } from '@kbn/lists-plugin/common/schemas/response/search_list_item_schema.mock';
import { filterEvents } from './filter_events';
import { partitionEvents } from './filter_events';
import { FieldSet } from './types';

describe('filterEvents', () => {
describe('partitionEvents', () => {
let listClient = listMock.getListClient();
let events = [sampleDocWithSortId('123', undefined, '1.1.1.1')];

Expand Down Expand Up @@ -43,11 +43,12 @@ describe('filterEvents', () => {
matchedSet: new Set([JSON.stringify(['1.1.1.1'])]),
},
];
const field = filterEvents({
const [included, excluded] = partitionEvents({
events,
fieldAndSetTuples,
});
expect([...field]).toEqual([]);
expect(included).toEqual([]);
expect(excluded).toEqual(events);
});

test('it does not filter out the event if it is "excluded"', () => {
Expand All @@ -59,11 +60,12 @@ describe('filterEvents', () => {
matchedSet: new Set([JSON.stringify(['1.1.1.1'])]),
},
];
const field = filterEvents({
const [included, excluded] = partitionEvents({
events,
fieldAndSetTuples,
});
expect([...field]).toEqual(events);
expect(included).toEqual(events);
expect(excluded).toEqual([]);
});

test('it does NOT filter out the event if the field is not found', () => {
Expand All @@ -75,11 +77,12 @@ describe('filterEvents', () => {
matchedSet: new Set([JSON.stringify(['1.1.1.1'])]),
},
];
const field = filterEvents({
const [included, excluded] = partitionEvents({
events,
fieldAndSetTuples,
});
expect([...field]).toEqual(events);
expect(included).toEqual(events);
expect(excluded).toEqual([]);
});

test('it does NOT filter out the event if it is in both an inclusion and exclusion list', () => {
Expand All @@ -100,10 +103,11 @@ describe('filterEvents', () => {
},
];

const field = filterEvents({
const [included, excluded] = partitionEvents({
events,
fieldAndSetTuples,
});
expect([...field]).toEqual(events);
expect(included).toEqual(events);
expect(excluded).toEqual([]);
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -5,19 +5,22 @@
* 2.0.
*/
import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { partition } from 'lodash';
import { FilterEventsOptions } from './types';

/**
* Check if for each tuple, the entry is not in both for when two or more value list entries exist.
* If the entry is in both an inclusion and exclusion list it will not be filtered out.
* @param events The events to check against
* @param fieldAndSetTuples The field and set tuples
* @returns A tuple where the first element is an array of alerts that should be created and second element is
* an array of alerts that matched the exception and should not be created.
*/
export const filterEvents = <T>({
export const partitionEvents = <T>({
events,
fieldAndSetTuples,
}: FilterEventsOptions<T>): Array<estypes.SearchHit<T>> => {
return events.filter((item) => {
}: FilterEventsOptions<T>): [Array<estypes.SearchHit<T>>, Array<estypes.SearchHit<T>>] => {
return partition(events, (item) => {
return fieldAndSetTuples
.map((tuple) => {
const eventItem = item.fields ? item.fields[tuple.field] : undefined;
Expand Down
Loading

0 comments on commit 5f290fc

Please sign in to comment.