From 75997d4cbb42b93dddb0ab431b8cf61118b32770 Mon Sep 17 00:00:00 2001 From: Stevan Buzejic Date: Mon, 9 Jan 2023 19:54:51 +0100 Subject: [PATCH] Changes from main applied Signed-off-by: Stevan Buzejic --- .../action/GetDetectorResponse.java | 1 - .../action/IndexDetectorResponse.java | 1 - .../alerts/AlertsService.java | 90 +- .../monitors/DetectorMonitorConfig.java | 12 +- .../findings/FindingsService.java | 66 +- .../securityanalytics/model/Detector.java | 187 +++-- .../model/DetectorInput.java | 45 +- .../transport/TransportGetAlertsAction.java | 6 +- .../transport/TransportGetFindingsAction.java | 9 +- .../TransportIndexDetectorAction.java | 249 +++--- src/main/resources/mappings/detectors.json | 8 + .../SecurityAnalyticsRestTestCase.java | 37 +- .../securityanalytics/TestHelpers.java | 57 +- .../action/IndexDetectorResponseTests.java | 7 +- .../alerts/AlertingServiceTests.java | 9 +- .../securityanalytics/alerts/AlertsIT.java | 773 +++++++++++++----- .../alerts/SecureAlertsRestApiIT.java | 3 +- .../securityanalytics/findings/FindingIT.java | 389 ++++++++- .../findings/FindingServiceTests.java | 9 +- .../findings/SecureFindingRestApiIT.java | 24 +- .../model/WriteableTests.java | 2 +- .../model/XContentTests.java | 8 +- .../resthandler/DetectorMonitorRestApiIT.java | 634 +++++++++++--- .../resthandler/DetectorRestApiIT.java | 48 +- .../resthandler/RuleRestApiIT.java | 8 +- 25 files changed, 2059 insertions(+), 623 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/action/GetDetectorResponse.java b/src/main/java/org/opensearch/securityanalytics/action/GetDetectorResponse.java index 6e166d5e5..78db2e0b8 100644 --- a/src/main/java/org/opensearch/securityanalytics/action/GetDetectorResponse.java +++ b/src/main/java/org/opensearch/securityanalytics/action/GetDetectorResponse.java @@ -62,7 +62,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws .field(_VERSION, version); builder.startObject("detector") .field(Detector.NAME_FIELD, detector.getName()) - .field(Detector.DETECTOR_TYPE_FIELD, detector.getDetectorType()) .field(Detector.ENABLED_FIELD, detector.getEnabled()) .field(Detector.SCHEDULE_FIELD, detector.getSchedule()) .field(Detector.INPUTS_FIELD, detector.getInputs()) diff --git a/src/main/java/org/opensearch/securityanalytics/action/IndexDetectorResponse.java b/src/main/java/org/opensearch/securityanalytics/action/IndexDetectorResponse.java index 78b0fb91f..a3dfc93ee 100644 --- a/src/main/java/org/opensearch/securityanalytics/action/IndexDetectorResponse.java +++ b/src/main/java/org/opensearch/securityanalytics/action/IndexDetectorResponse.java @@ -57,7 +57,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws .field(_VERSION, version); builder.startObject("detector") .field(Detector.NAME_FIELD, detector.getName()) - .field(Detector.DETECTOR_TYPE_FIELD, detector.getDetectorType()) .field(Detector.ENABLED_FIELD, detector.getEnabled()) .field(Detector.SCHEDULE_FIELD, detector.getSchedule()) .field(Detector.INPUTS_FIELD, detector.getInputs()) diff --git a/src/main/java/org/opensearch/securityanalytics/alerts/AlertsService.java b/src/main/java/org/opensearch/securityanalytics/alerts/AlertsService.java index d14a17344..ff90a8bfc 100644 --- a/src/main/java/org/opensearch/securityanalytics/alerts/AlertsService.java +++ b/src/main/java/org/opensearch/securityanalytics/alerts/AlertsService.java @@ -80,28 +80,46 @@ public void onResponse(GetDetectorResponse getDetectorResponse) { detector.getMonitorIds().forEach( monitorId -> monitorToDetectorMapping.put(monitorId, detector.getId()) ); - // Get alerts for all monitor ids - AlertsService.this.getAlertsByMonitorIds( + + List detectorTypes = detector.getDetectorTypes(); + + GroupedActionListener getAlertsResponseListener = new GroupedActionListener( + new ActionListener>() { + @Override + public void onResponse(Collection alertsResponses) { + List alerts = new ArrayList<>(); + // Merge all findings into one response + int totalAlerts = alertsResponses.stream().map(GetAlertsResponse::getTotalAlerts).collect( + Collectors.summingInt(Integer::intValue)); + alerts.addAll(alertsResponses.stream().flatMap(getAlertsResponse -> getAlertsResponse.getAlerts().stream()).collect( + Collectors.toList())); + + GetAlertsResponse masterResponse = new GetAlertsResponse( + alerts, + totalAlerts + ); + listener.onResponse(masterResponse); + } + @Override + public void onFailure(Exception e) { + log.error("Failed to fetch alerts for detectorId: " + detectorId, e); + listener.onFailure(SecurityAnalyticsException.wrap(e)); + } + }, detectorTypes.size()); + + for (String detectorType: detectorTypes) { + // Get alerts for all monitor ids + AlertsService.this.getAlertsByMonitorIds( monitorToDetectorMapping, + // TODO - Monitor list will contain all the monitors event those from another detector type monitorIds, - DetectorMonitorConfig.getAllAlertsIndicesPattern(detector.getDetectorType()), + DetectorMonitorConfig.getAllAlertsIndicesPattern(detectorType), table, severityLevel, alertState, - new ActionListener<>() { - @Override - public void onResponse(GetAlertsResponse getAlertsResponse) { - // Send response back - listener.onResponse(getAlertsResponse); - } - - @Override - public void onFailure(Exception e) { - log.error("Failed to fetch alerts for detectorId: " + detectorId, e); - listener.onFailure(SecurityAnalyticsException.wrap(e)); - } - } - ); + getAlertsResponseListener + ); + } } @Override @@ -246,18 +264,46 @@ public void getAlerts(List alertIds, Detector detector, Table table, ActionListener actionListener) { - GetAlertsRequest request = new GetAlertsRequest( + + List detectorTypes = detector.getDetectorTypes(); + + ActionListener getAlertsResponseListener = new GroupedActionListener( + new ActionListener>() { + @Override + public void onResponse(Collection alertsResponses) { + List alerts = new ArrayList<>(); + // Merge all findings into one response + int totalAlerts = alertsResponses.stream().map(org.opensearch.commons.alerting.action.GetAlertsResponse::getTotalAlerts).collect( + Collectors.summingInt(Integer::intValue)); + alerts.addAll(alertsResponses.stream().flatMap(getAlertsResponse -> getAlertsResponse.getAlerts().stream()).collect( + Collectors.toList())); + + org.opensearch.commons.alerting.action.GetAlertsResponse masterResponse = new org.opensearch.commons.alerting.action.GetAlertsResponse( + alerts, + totalAlerts + ); + actionListener.onResponse(masterResponse); + } + @Override + public void onFailure(Exception e) { + log.error("Failed to fetch alerts for detectorId: " + detector.getId(), e); + actionListener.onFailure(SecurityAnalyticsException.wrap(e)); + } + }, detectorTypes.size()); + + for(String detectorType: detectorTypes) { + GetAlertsRequest request = new GetAlertsRequest( table, "ALL", "ALL", null, - DetectorMonitorConfig.getAllAlertsIndicesPattern(detector.getDetectorType()), + DetectorMonitorConfig.getAllAlertsIndicesPattern(detectorType), null, alertIds); - AlertingPluginInterface.INSTANCE.getAlerts( + AlertingPluginInterface.INSTANCE.getAlerts( (NodeClient) client, - request, actionListener); - + request, getAlertsResponseListener); + } } /** diff --git a/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java b/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java index fc79c6454..d8eaf8671 100644 --- a/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java +++ b/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java @@ -5,6 +5,8 @@ package org.opensearch.securityanalytics.config.monitors; import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; import org.opensearch.securityanalytics.model.Detector; @@ -15,7 +17,7 @@ public class DetectorMonitorConfig { - + private static Pattern findingIndexRegexPattern = Pattern.compile(".opensearch-sap-(.*?)-findings"); public static final String OPENSEARCH_DEFAULT_RULE_INDEX = ".opensearch-sap-detectors-queries-default"; public static final String OPENSEARCH_DEFAULT_ALERT_INDEX = ".opensearch-sap-alerts-default"; public static final String OPENSEARCH_DEFAULT_ALL_ALERT_INDICES_PATTERN = ".opensearch-sap-alerts-default*"; @@ -128,7 +130,7 @@ public static String getFindingsIndexPattern(String detectorType) { OPENSEARCH_DEFAULT_FINDINGS_INDEX_PATTERN; } - public static Map> getRuleIndexMappingsByType(String detectorType) { + public static Map> getRuleIndexMappingsByType() { HashMap properties = new HashMap<>(); properties.put("analyzer", "rule_analyzer"); HashMap> fieldMappingProperties = new HashMap<>(); @@ -136,6 +138,12 @@ public static Map> getRuleIndexMappingsByType(String return fieldMappingProperties; } + public static String getRuleCategoryFromFindingIndexName(String findingIndex) { + Matcher matcher = findingIndexRegexPattern.matcher(findingIndex); + matcher.find(); + return matcher.group(1); + } + public static class MonitorConfig { private final String alertsIndex; private final String alertsHistoryIndex; diff --git a/src/main/java/org/opensearch/securityanalytics/findings/FindingsService.java b/src/main/java/org/opensearch/securityanalytics/findings/FindingsService.java index dc5880ec6..a30e2f585 100644 --- a/src/main/java/org/opensearch/securityanalytics/findings/FindingsService.java +++ b/src/main/java/org/opensearch/securityanalytics/findings/FindingsService.java @@ -5,6 +5,7 @@ package org.opensearch.securityanalytics.findings; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -14,6 +15,7 @@ import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchStatusException; import org.opensearch.action.ActionListener; +import org.opensearch.action.support.GroupedActionListener; import org.opensearch.client.Client; import org.opensearch.client.node.NodeClient; import org.opensearch.commons.alerting.AlertingPluginInterface; @@ -52,7 +54,7 @@ public FindingsService(Client client) { * @param table group of search related parameters * @param listener ActionListener to get notified on response or error */ - public void getFindingsByDetectorId(String detectorId, Table table, ActionListener listener ) { + public void getFindingsByDetectorId(String detectorId, Table table, ActionListener listener) { this.client.execute(GetDetectorAction.INSTANCE, new GetDetectorRequest(detectorId, -3L), new ActionListener<>() { @Override @@ -60,43 +62,50 @@ public void onResponse(GetDetectorResponse getDetectorResponse) { // Get all monitor ids from detector Detector detector = getDetectorResponse.getDetector(); List monitorIds = detector.getMonitorIds(); - ActionListener getFindingsResponseListener = new ActionListener<>() { - @Override - public void onResponse(GetFindingsResponse resp) { - Integer totalFindings = 0; - List findings = new ArrayList<>(); - // Merge all findings into one response - totalFindings += resp.getTotalFindings(); - findings.addAll(resp.getFindings()); - - GetFindingsResponse masterResponse = new GetFindingsResponse( - totalFindings, - findings - ); - // Send master response back - listener.onResponse(masterResponse); - } - - @Override - public void onFailure(Exception e) { - log.error("Failed to fetch findings for detector " + detectorId, e); - listener.onFailure(SecurityAnalyticsException.wrap(e)); - } - }; // monitor --> detectorId mapping Map monitorToDetectorMapping = new HashMap<>(); detector.getMonitorIds().forEach( monitorId -> monitorToDetectorMapping.put(monitorId, detector) ); + + List detectorTypes = detector.getDetectorTypes(); + + ActionListener getFindingsResponseListener = new GroupedActionListener( + new ActionListener>() { + @Override + public void onResponse(Collection findingsResponses) { + List findings = new ArrayList<>(); + // Merge all findings into one response + int totalFindings = findingsResponses.stream().map(GetFindingsResponse::getTotalFindings).collect( + Collectors.summingInt(Integer::intValue)); + findings.addAll(findingsResponses.stream().flatMap(getFindingsResponse -> getFindingsResponse.getFindings().stream()).collect( + Collectors.toList())); + + GetFindingsResponse masterResponse = new GetFindingsResponse( + totalFindings, + findings + ); + listener.onResponse(masterResponse); + } + @Override + public void onFailure(Exception e) { + log.error("Failed to fetch findings for detector " + detectorId, e); + listener.onFailure(SecurityAnalyticsException.wrap(e)); + } + }, detectorTypes.size()); + // Get findings for all monitor ids - FindingsService.this.getFindingsByMonitorIds( + for (String detectorType: detectorTypes) { + FindingsService.this.getFindingsByMonitorIds( monitorToDetectorMapping, monitorIds, - DetectorMonitorConfig.getAllFindingsIndicesPattern(detector.getDetectorType()), + DetectorMonitorConfig.getAllFindingsIndicesPattern(detectorType), table, getFindingsResponseListener - ); + ); + } + } @Override @@ -136,6 +145,7 @@ public void getFindingsByMonitorIds( public void onResponse( org.opensearch.commons.alerting.action.GetFindingsResponse getFindingsResponse ) { + log.error("alerts response size from alerting" + getFindingsResponse.getTotalFindings()); // Convert response to SA's GetFindingsResponse listener.onResponse(new GetFindingsResponse( getFindingsResponse.getTotalFindings(), @@ -207,7 +217,7 @@ public void onFailure(Exception e) { public FindingDto mapFindingWithDocsToFindingDto(FindingWithDocs findingWithDocs, Detector detector) { List docLevelQueries = findingWithDocs.getFinding().getDocLevelQueries(); if (docLevelQueries.isEmpty()) { // this is finding generated by a bucket level monitor - for (Map.Entry entry : detector.getRuleIdMonitorIdMap().entrySet()) { + for (Map.Entry entry : detector.getBucketRuleIdMonitorIdMap().entrySet()) { if(entry.getValue().equals(findingWithDocs.getFinding().getMonitorId())) { docLevelQueries = Collections.singletonList(new DocLevelQuery(entry.getKey(),"","",Collections.emptyList())); } diff --git a/src/main/java/org/opensearch/securityanalytics/model/Detector.java b/src/main/java/org/opensearch/securityanalytics/model/Detector.java index a05f04b81..637e1513d 100644 --- a/src/main/java/org/opensearch/securityanalytics/model/Detector.java +++ b/src/main/java/org/opensearch/securityanalytics/model/Detector.java @@ -5,7 +5,12 @@ package org.opensearch.securityanalytics.model; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.function.Function; +import org.apache.commons.lang3.tuple.Pair; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.common.ParseField; @@ -18,6 +23,7 @@ import org.opensearch.common.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentParser; import org.opensearch.common.xcontent.XContentParserUtils; +import org.opensearch.commons.alerting.action.IndexMonitorRequest; import org.opensearch.commons.alerting.model.CronSchedule; import org.opensearch.commons.alerting.model.Schedule; import org.opensearch.commons.authuser.User; @@ -32,6 +38,7 @@ import java.util.Objects; import java.util.stream.Collectors; +import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig; public class Detector implements Writeable, ToXContentObject { @@ -53,6 +60,8 @@ public class Detector implements Writeable, ToXContentObject { public static final String ALERTING_MONITOR_ID = "monitor_id"; public static final String BUCKET_MONITOR_ID_RULE_ID = "bucket_monitor_id_rule_id"; + + public static final String DOC_MONITOR_ID_PER_CATEGORY = "doc_monitor_id_per_category"; private static final String RULE_TOPIC_INDEX = "rule_topic_index"; private static final String ALERTS_INDEX = "alert_index"; @@ -87,8 +96,6 @@ public class Detector implements Writeable, ToXContentObject { private Instant enabledTime; - private DetectorType detectorType; - private User user; private List inputs; @@ -97,7 +104,9 @@ public class Detector implements Writeable, ToXContentObject { private List monitorIds; - private Map ruleIdMonitorIdMap; + private Map bucketRuleIdMonitorIdMap; + + private Map docLevelMonitorPerCategory; private String ruleIndex; @@ -114,10 +123,10 @@ public class Detector implements Writeable, ToXContentObject { private final String type; public Detector(String id, Long version, String name, Boolean enabled, Schedule schedule, - Instant lastUpdateTime, Instant enabledTime, DetectorType detectorType, - User user, List inputs, List triggers, List monitorIds, - String ruleIndex, String alertsIndex, String alertsHistoryIndex, String alertsHistoryIndexPattern, - String findingsIndex, String findingsIndexPattern, Map rulePerMonitor) { + Instant lastUpdateTime, Instant enabledTime, + User user, List inputs, List triggers, List monitorIds, + String ruleIndex, String alertsIndex, String alertsHistoryIndex, String alertsHistoryIndexPattern, + String findingsIndex, String findingsIndexPattern, Map rulePerMonitor, Map docLevelMonitorPerCategory) { this.type = DETECTOR_TYPE; this.id = id != null ? id : NO_ID; @@ -127,7 +136,6 @@ public Detector(String id, Long version, String name, Boolean enabled, Schedule this.schedule = schedule; this.lastUpdateTime = lastUpdateTime; this.enabledTime = enabledTime; - this.detectorType = detectorType; this.user = user; this.inputs = inputs; this.triggers = triggers; @@ -138,7 +146,8 @@ public Detector(String id, Long version, String name, Boolean enabled, Schedule this.alertsHistoryIndexPattern = alertsHistoryIndexPattern; this.findingsIndex = findingsIndex; this.findingsIndexPattern = findingsIndexPattern; - this.ruleIdMonitorIdMap = rulePerMonitor; + this.bucketRuleIdMonitorIdMap = rulePerMonitor; + this.docLevelMonitorPerCategory = docLevelMonitorPerCategory; if (enabled) { Objects.requireNonNull(enabledTime); @@ -154,7 +163,6 @@ public Detector(StreamInput sin) throws IOException { Schedule.readFrom(sin), sin.readInstant(), sin.readOptionalInstant(), - sin.readEnum(DetectorType.class), sin.readBoolean() ? new User(sin) : null, sin.readList(DetectorInput::readFrom), sin.readList(DetectorTrigger::readFrom), @@ -165,6 +173,7 @@ public Detector(StreamInput sin) throws IOException { sin.readString(), sin.readString(), sin.readString(), + sin.readMap(StreamInput::readString, StreamInput::readString), sin.readMap(StreamInput::readString, StreamInput::readString) ); } @@ -183,7 +192,6 @@ public void writeTo(StreamOutput out) throws IOException { schedule.writeTo(out); out.writeInstant(lastUpdateTime); out.writeOptionalInstant(enabledTime); - out.writeEnum(detectorType); out.writeBoolean(user != null); if (user != null) { user.writeTo(out); @@ -199,7 +207,8 @@ public void writeTo(StreamOutput out) throws IOException { out.writeStringCollection(monitorIds); out.writeString(ruleIndex); - out.writeMap(ruleIdMonitorIdMap, StreamOutput::writeString, StreamOutput::writeString); + out.writeMap(bucketRuleIdMonitorIdMap, StreamOutput::writeString, StreamOutput::writeString); + out.writeMap(docLevelMonitorPerCategory, StreamOutput::writeString, StreamOutput::writeString); } public XContentBuilder toXContentWithUser(XContentBuilder builder, Params params) throws IOException { @@ -247,8 +256,7 @@ private XContentBuilder createXContentBuilder(XContentBuilder builder, ToXConten builder.startObject(type); } builder.field(TYPE_FIELD, type) - .field(NAME_FIELD, name) - .field(DETECTOR_TYPE_FIELD, detectorType.getDetectorType()); + .field(NAME_FIELD, name); if (!secure) { if (user == null) { @@ -283,7 +291,8 @@ private XContentBuilder createXContentBuilder(XContentBuilder builder, ToXConten } builder.field(ALERTING_MONITOR_ID, monitorIds); - builder.field(BUCKET_MONITOR_ID_RULE_ID, ruleIdMonitorIdMap); + builder.field(BUCKET_MONITOR_ID_RULE_ID, bucketRuleIdMonitorIdMap); + builder.field(DOC_MONITOR_ID_PER_CATEGORY, docLevelMonitorPerCategory); builder.field(RULE_TOPIC_INDEX, ruleIndex); builder.field(ALERTS_INDEX, alertsIndex); builder.field(ALERTS_HISTORY_INDEX, alertsHistoryIndex); @@ -329,6 +338,7 @@ public static Detector parse(XContentParser xcp, String id, Long version) throws List triggers = new ArrayList<>(); List monitorIds = new ArrayList<>(); Map rulePerMonitor = new HashMap<>(); + Map docLevelRulePerCategory = new HashMap<>(); String ruleIndex = null; String alertsIndex = null; @@ -337,6 +347,8 @@ public static Detector parse(XContentParser xcp, String id, Long version) throws String findingsIndex = null; String findingsIndexPattern = null; + List allowedTypes = Arrays.stream(DetectorType.values()).map(DetectorType::getDetectorType).collect(Collectors.toList()); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { String fieldName = xcp.currentName(); @@ -347,11 +359,23 @@ public static Detector parse(XContentParser xcp, String id, Long version) throws name = xcp.text(); break; case DETECTOR_TYPE_FIELD: - detectorType = xcp.text(); - List allowedTypes = Arrays.stream(DetectorType.values()).map(DetectorType::getDetectorType).collect(Collectors.toList()); - - if (!allowedTypes.contains(detectorType.toLowerCase(Locale.ROOT))) { - throw new IllegalArgumentException(String.format(Locale.getDefault(), "Detector type should be one of %s", allowedTypes)); + if (xcp.currentToken() != XContentParser.Token.VALUE_NULL) { + detectorType = xcp.text(); + if (!allowedTypes.contains(detectorType.toLowerCase(Locale.ROOT))) { + throw new IllegalArgumentException(String.format(Locale.getDefault(), + "Detector type should be one of %s", + allowedTypes)); + } + // In order to keep the existing detector types and to be backward compatible + if (detectorType != null && !inputs.get(0).getDetectorTypes().contains(DetectorType.valueOf(detectorType))) { + inputs.get(0).getDetectorTypes().add(DetectorType.valueOf(detectorType.toUpperCase(Locale.ROOT))); + } + // Added on both places since not sure if there is an order in which fields are being parsed + // In order to be backward compatible - if there is doc_level_monitor key, that means that we had one detectorType supported + // Re-map from { 1 : docLevelMonitorId } to { rule_category: docLevelMonitorId } + if(rulePerMonitor.containsKey(DOC_LEVEL_MONITOR) && detectorType != null && !rulePerMonitor.containsKey(detectorType)) { + docLevelRulePerCategory.put(detectorType, rulePerMonitor.get(DOC_LEVEL_MONITOR)); + } } break; case USER_FIELD: @@ -373,6 +397,10 @@ public static Detector parse(XContentParser xcp, String id, Long version) throws DetectorInput input = DetectorInput.parse(xcp); inputs.add(input); } + // In order to keep the existing detector types and to be backward compatible + if (detectorType != null && !inputs.get(0).getDetectorTypes().contains(DetectorType.valueOf(detectorType.toUpperCase(Locale.ROOT)))) { + inputs.get(0).getDetectorTypes().add(DetectorType.valueOf(detectorType.toUpperCase(Locale.ROOT))); + } break; case TRIGGERS_FIELD: XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); @@ -409,7 +437,22 @@ public static Detector parse(XContentParser xcp, String id, Long version) throws } break; case BUCKET_MONITOR_ID_RULE_ID: - rulePerMonitor= xcp.mapStrings(); + rulePerMonitor = xcp.mapStrings(); + // In order to be backward compatible - if there is doc_level_monitor key, that means that we had one detectorType supported + // Re-map from { 1 : docLevelMonitorId } to { rule_category: docLevelMonitorId } + if(rulePerMonitor.containsKey(DOC_LEVEL_MONITOR) && detectorType != null && !rulePerMonitor.containsKey(detectorType)) { + rulePerMonitor.put(detectorType, rulePerMonitor.get(DOC_LEVEL_MONITOR)); + } + + break; + case DOC_MONITOR_ID_PER_CATEGORY: + docLevelRulePerCategory = xcp.mapStrings(); + + // In order to be backward compatible - if there is doc_level_monitor key, that means that we had one detectorType supported + // Re-map from { 1 : docLevelMonitorId } to { rule_category: docLevelMonitorId } + if(rulePerMonitor.containsKey(DOC_LEVEL_MONITOR) && detectorType != null && !rulePerMonitor.containsKey(detectorType)) { + docLevelRulePerCategory.put(detectorType, rulePerMonitor.get(DOC_LEVEL_MONITOR)); + } break; case RULE_TOPIC_INDEX: ruleIndex = xcp.text(); @@ -448,7 +491,6 @@ public static Detector parse(XContentParser xcp, String id, Long version) throws Objects.requireNonNull(schedule, "Detector schedule is null"), lastUpdateTime != null ? lastUpdateTime : Instant.now(), enabledTime, - DetectorType.valueOf(detectorType.toUpperCase(Locale.ROOT)), user, inputs, triggers, @@ -459,7 +501,9 @@ public static Detector parse(XContentParser xcp, String id, Long version) throws alertsHistoryIndexPattern, findingsIndex, findingsIndexPattern, - rulePerMonitor); + rulePerMonitor, + docLevelRulePerCategory + ); } public static Detector readFrom(StreamInput sin) throws IOException { @@ -494,10 +538,6 @@ public Instant getEnabledTime() { return enabledTime; } - public String getDetectorType() { - return detectorType.getDetectorType(); - } - public User getUser() { return user; } @@ -514,24 +554,17 @@ public String getRuleIndex() { return ruleIndex; } - public String getAlertsIndex() { - return alertsIndex; - } - - public String getAlertsHistoryIndex() { - return alertsHistoryIndex; - } - - public String getAlertsHistoryIndexPattern() { - return alertsHistoryIndexPattern; - } - - public String getFindingsIndex() { - return findingsIndex; + public List getDetectorTypes() { + // In the case of detectors created before support of multiple detector types + if(inputs == null || inputs.isEmpty() || inputs.get(0).getDetectorTypes().isEmpty()) { + return Collections.emptyList(); + } + return inputs.get(0).getDetectorTypes().stream().map(DetectorType::getDetectorType).collect(Collectors.toList()); } - public String getFindingsIndexPattern() { - return findingsIndexPattern; + public List getRuleIndices() { + return getDetectorTypes().stream().map(detectorType -> DetectorMonitorConfig.getRuleIndex(detectorType)).collect( + Collectors.toList()); } public List getMonitorIds() { @@ -542,7 +575,13 @@ public void setUser(User user) { this.user = user; } - public Map getRuleIdMonitorIdMap() {return ruleIdMonitorIdMap; } + public Map getBucketRuleIdMonitorIdMap() {return bucketRuleIdMonitorIdMap; } + + public Map getDocLevelMonitorPerCategory() {return docLevelMonitorPerCategory;} + + public String getDocLevelMonitorIdForRuleCategory (String ruleCategory) { + return docLevelMonitorPerCategory.get(ruleCategory); + } public void setId(String id) { this.id = id; @@ -552,34 +591,10 @@ public void setVersion(Long version) { this.version = version; } - public void setRuleIndex(String ruleIndex) { - this.ruleIndex = ruleIndex; - } - - public void setAlertsIndex(String alertsIndex) { - this.alertsIndex = alertsIndex; - } - - public void setAlertsHistoryIndex(String alertsHistoryIndex) { - this.alertsHistoryIndex = alertsHistoryIndex; - } - - public void setAlertsHistoryIndexPattern(String alertsHistoryIndexPattern) { - this.alertsHistoryIndexPattern = alertsHistoryIndexPattern; - } - public void setEnabledTime(Instant enabledTime) { this.enabledTime = enabledTime; } - public void setFindingsIndex(String findingsIndex) { - this.findingsIndex = findingsIndex; - } - - public void setFindingsIndexPattern(String findingsIndexPattern) { - this.findingsIndexPattern = findingsIndexPattern; - } - public void setLastUpdateTime(Instant lastUpdateTime) { this.lastUpdateTime = lastUpdateTime; } @@ -591,12 +606,36 @@ public void setInputs(List inputs) { public void setMonitorIds(List monitorIds) { this.monitorIds = monitorIds; } - public void setRuleIdMonitorIdMap(Map ruleIdMonitorIdMap) { - this.ruleIdMonitorIdMap = ruleIdMonitorIdMap; + public void setBucketRuleIdMonitorIdMap(Map bucketRuleIdMonitorIdMap) { + this.bucketRuleIdMonitorIdMap = bucketRuleIdMonitorIdMap; + } + + public void setDocLevelMonitorPerCategory(Map docLevelMonitorPerCategory) { + this.docLevelMonitorPerCategory = docLevelMonitorPerCategory; + } + + public List getMonitorIdsToBeDeleted(List monitorsToBeUpdated) { + List monitorIdsToBeDeleted = bucketRuleIdMonitorIdMap.values().stream().collect(Collectors.toList()); + monitorIdsToBeDeleted.addAll(docLevelMonitorPerCategory.values().stream().collect(Collectors.toList())); + + List monitorIds = monitorsToBeUpdated.stream().map(IndexMonitorRequest::getMonitorId).collect( + Collectors.toList()); + monitorIdsToBeDeleted.removeAll(monitorIds); + + return monitorIdsToBeDeleted; } - public String getDocLevelMonitorId() { - return ruleIdMonitorIdMap.get(DOC_LEVEL_MONITOR); + public List getRuleTopicsForMonitors(List monitorIds) { + Set monitorIdsSet = new HashSet<>(monitorIds); + List ruleTopics = new ArrayList<>(); + + for (Entry categoryDocMonitorId: docLevelMonitorPerCategory.entrySet()) { + if (monitorIdsSet.contains(categoryDocMonitorId.getValue())) { + String ruleCategory = categoryDocMonitorId.getKey(); + ruleTopics.add(DetectorMonitorConfig.getRuleIndex(ruleCategory)); + } + } + return ruleTopics; } @Override @@ -604,11 +643,11 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Detector detector = (Detector) o; - return Objects.equals(id, detector.id) && Objects.equals(version, detector.version) && Objects.equals(name, detector.name) && Objects.equals(enabled, detector.enabled) && Objects.equals(schedule, detector.schedule) && Objects.equals(lastUpdateTime, detector.lastUpdateTime) && Objects.equals(enabledTime, detector.enabledTime) && detectorType == detector.detectorType && ((user == null && detector.user == null) || Objects.equals(user, detector.user)) && Objects.equals(inputs, detector.inputs) && Objects.equals(triggers, detector.triggers) && Objects.equals(type, detector.type) && Objects.equals(monitorIds, detector.monitorIds) && Objects.equals(ruleIndex, detector.ruleIndex); + return Objects.equals(id, detector.id) && Objects.equals(version, detector.version) && Objects.equals(name, detector.name) && Objects.equals(enabled, detector.enabled) && Objects.equals(schedule, detector.schedule) && Objects.equals(lastUpdateTime, detector.lastUpdateTime) && Objects.equals(enabledTime, detector.enabledTime) && ((user == null && detector.user == null) || Objects.equals(user, detector.user)) && Objects.equals(inputs, detector.inputs) && Objects.equals(triggers, detector.triggers) && Objects.equals(type, detector.type) && Objects.equals(monitorIds, detector.monitorIds) && Objects.equals(ruleIndex, detector.ruleIndex); } @Override public int hashCode() { - return Objects.hash(id, version, name, enabled, schedule, lastUpdateTime, enabledTime, detectorType, user, inputs, triggers, type, monitorIds, ruleIndex); + return Objects.hash(id, version, name, enabled, schedule, lastUpdateTime, enabledTime, user, inputs, triggers, type, monitorIds, ruleIndex); } } diff --git a/src/main/java/org/opensearch/securityanalytics/model/DetectorInput.java b/src/main/java/org/opensearch/securityanalytics/model/DetectorInput.java index 0f7b7ef10..8cada8efd 100644 --- a/src/main/java/org/opensearch/securityanalytics/model/DetectorInput.java +++ b/src/main/java/org/opensearch/securityanalytics/model/DetectorInput.java @@ -4,6 +4,9 @@ */ package org.opensearch.securityanalytics.model; +import java.util.Arrays; +import java.util.Collections; +import java.util.Locale; import org.opensearch.common.ParseField; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; @@ -20,6 +23,7 @@ import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; +import org.opensearch.securityanalytics.model.Detector.DetectorType; public class DetectorInput implements Writeable, ToXContentObject { @@ -31,6 +35,8 @@ public class DetectorInput implements Writeable, ToXContentObject { private List prePackagedRules; + private List detectorTypes; + private static final String NO_DESCRIPTION = ""; protected static final String DESCRIPTION_FIELD = "description"; @@ -39,17 +45,20 @@ public class DetectorInput implements Writeable, ToXContentObject { protected static final String CUSTOM_RULES_FIELD = "custom_rules"; protected static final String PREPACKAGED_RULES_FIELD = "pre_packaged_rules"; + protected static final String DETECTOR_TYPES_FIELD = "detector_types"; + public static final NamedXContentRegistry.Entry XCONTENT_REGISTRY = new NamedXContentRegistry.Entry( DetectorInput.class, new ParseField(DETECTOR_INPUT_FIELD), DetectorInput::parse ); - public DetectorInput(String description, List indices, List customRules, List prePackagedRules) { + public DetectorInput(String description, List indices, List customRules, List prePackagedRules, List detectorTypes) { this.description = description; this.indices = indices; this.customRules = customRules; this.prePackagedRules = prePackagedRules; + this.detectorTypes = detectorTypes; } public DetectorInput(StreamInput sin) throws IOException { @@ -57,7 +66,8 @@ public DetectorInput(StreamInput sin) throws IOException { sin.readString(), sin.readStringList(), sin.readList(DetectorRule::new), - sin.readList(DetectorRule::new) + sin.readList(DetectorRule::new), + sin.readStringList().stream().map(s -> DetectorType.valueOf(s.toUpperCase(Locale.ROOT))).collect(Collectors.toList()) ); } @@ -72,10 +82,14 @@ public Map asTemplateArg() { @Override public void writeTo(StreamOutput out) throws IOException { + List detectorTypes = getDetectorTypes().stream().map(detectorType -> detectorType.getDetectorType()).collect( + Collectors.toList()); + out.writeString(description); out.writeStringCollection(indices); out.writeCollection(customRules); out.writeCollection(prePackagedRules); + out.writeStringCollection(detectorTypes); } @Override @@ -89,12 +103,15 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws DetectorRule[] prePackagedRulesArray = new DetectorRule[]{}; prePackagedRulesArray = prePackagedRules.toArray(prePackagedRulesArray); + String [] detectorTypesArray = detectorTypes.stream().map(DetectorType::getDetectorType).toArray(String[]::new); + builder.startObject() .startObject(DETECTOR_INPUT_FIELD) .field(DESCRIPTION_FIELD, description) .field(INDICES_FIELD, indicesArray) .field(CUSTOM_RULES_FIELD, customRulesArray) .field(PREPACKAGED_RULES_FIELD, prePackagedRulesArray) + .field(DETECTOR_TYPES_FIELD, detectorTypesArray) .endObject() .endObject(); return builder; @@ -105,6 +122,7 @@ public static DetectorInput parse(XContentParser xcp) throws IOException { List indices = new ArrayList<>(); List customRules = new ArrayList<>(); List prePackagedRules = new ArrayList<>(); + List detectorTypes = new ArrayList<>(); XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, xcp.nextToken(), xcp); @@ -131,6 +149,12 @@ public static DetectorInput parse(XContentParser xcp) throws IOException { customRules.add(DetectorRule.parse(xcp)); } break; + case DETECTOR_TYPES_FIELD: + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + detectorTypes.add(DetectorType.valueOf(xcp.text().toUpperCase(Locale.ROOT))); + } + break; case PREPACKAGED_RULES_FIELD: XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { @@ -139,7 +163,7 @@ public static DetectorInput parse(XContentParser xcp) throws IOException { } } XContentParserUtils.ensureExpectedToken(XContentParser.Token.END_OBJECT, xcp.nextToken(), xcp); - return new DetectorInput(description, indices, customRules, prePackagedRules); + return new DetectorInput(description, indices, customRules, prePackagedRules, detectorTypes); } public static DetectorInput readFrom(StreamInput sin) throws IOException { @@ -150,6 +174,10 @@ public void setCustomRules(List customRules) { this.customRules = customRules; } + public void setDetectorTypes(List detectorTypes) { + this.detectorTypes = detectorTypes; + } + public String getDescription() { return description; } @@ -166,6 +194,17 @@ public List getPrePackagedRules() { return prePackagedRules; } + public List getDetectorTypes () { + return detectorTypes; + } + + public void addDetectorType(String detectorType) { + List allowedTypes = Arrays.stream(DetectorType.values()).map(DetectorType::getDetectorType).collect(Collectors.toList()); + if (!allowedTypes.contains(detectorType)) { + throw new IllegalArgumentException(String.format(Locale.getDefault(), "Detector type should be one of %s", allowedTypes)); + } + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportGetAlertsAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportGetAlertsAction.java index 2eefc0a03..beee3f559 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportGetAlertsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportGetAlertsAction.java @@ -43,6 +43,8 @@ public class TransportGetAlertsAction extends HandledTransportAction implements SecureTransportAction { + public static final String DETECTOR_INPUT_PATH = "detector.inputs.detector_input"; + public static final String DETECTOR_TYPES = "detector.inputs.detector_input.detector_types"; private final TransportSearchDetectorAction transportSearchDetectorAction; private final NamedXContentRegistry xContentRegistry; @@ -96,10 +98,10 @@ protected void doExecute(Task task, GetAlertsRequest request, ActionListener implements SecureTransportAction { + public static final String DETECTOR_INPUT_PATH = "detector.inputs.detector_input"; + public static final String DETECTOR_TYPES = "detector.inputs.detector_input.detector_types"; private final TransportSearchDetectorAction transportSearchDetectorAction; private final NamedXContentRegistry xContentRegistry; @@ -99,10 +98,10 @@ protected void doExecute(Task task, GetFindingsRequest request, ActionListener> rulesById, Detector detector, ActionListener> listener, WriteRequest.RefreshPolicy refreshPolicy) throws SigmaError, IOException { + // TODO - should we check if the detector type list is the same like list of rule categories? + /**List ruleCategories = rulesById.stream().map(ruleIdRulePair -> ruleIdRulePair.getRight().getCategory()).distinct().collect(Collectors.toList()); + if (detector.getDetectorTypes().size() != ruleCategories.size() || + detector.getDetectorTypes().containsAll(ruleCategories) == false) { + listener.onFailure(new IllegalArgumentException("Detector types and rule categories are not the same")); + return; + }*/ + List> docLevelRules = rulesById.stream().filter(it -> !it.getRight().isAggregationRule()).collect( Collectors.toList()); List> bucketLevelRules = rulesById.stream().filter(it -> it.getRight().isAggregationRule()).collect( @@ -234,7 +256,7 @@ private void createMonitorFromQueries(String index, List> rul List monitorRequests = new ArrayList<>(); if (!docLevelRules.isEmpty()) { - monitorRequests.add(createDocLevelMonitorRequest(Pair.of(index, docLevelRules), detector, refreshPolicy, Monitor.NO_ID, Method.POST)); + monitorRequests.addAll(createDocLevelMonitorRequests(index, docLevelRules, detector, refreshPolicy, Monitor.NO_ID, Method.POST)); } if (!bucketLevelRules.isEmpty()) { monitorRequests.addAll(buildBucketLevelMonitorRequests(Pair.of(index, bucketLevelRules), detector, refreshPolicy, Monitor.NO_ID, Method.POST)); @@ -295,14 +317,14 @@ private void updateMonitorFromQueries(String index, List> rul } // Pair of RuleId - MonitorId for existing monitors of the detector - Map monitorPerRule = detector.getRuleIdMonitorIdMap(); + Map bucketRuleIdMonitorId = detector.getBucketRuleIdMonitorIdMap(); for (Pair query: bucketLevelRules) { Rule rule = query.getRight(); if (rule.getAggregationQueries() != null){ // Detect if the monitor should be added or updated - if (monitorPerRule.containsKey(rule.getId())) { - String monitorId = monitorPerRule.get(rule.getId()); + if (bucketRuleIdMonitorId.containsKey(rule.getId())) { + String monitorId = bucketRuleIdMonitorId.get(rule.getId()); monitorsToBeUpdated.add(createBucketLevelMonitorRequest(query.getRight(), index, detector, @@ -323,25 +345,29 @@ private void updateMonitorFromQueries(String index, List> rul } } - List> docLevelRules = rulesById.stream().filter(it -> !it.getRight().isAggregationRule()).collect( - Collectors.toList()); + Map>> docLevelRulesByCategory = rulesById.stream().filter(it -> !it.getRight().isAggregationRule()).collect( + Collectors.groupingBy(it -> it.getRight().getCategory())); // Process doc level monitors - if (!docLevelRules.isEmpty()) { - if (detector.getDocLevelMonitorId() == null) { - monitorsToBeAdded.add(createDocLevelMonitorRequest(Pair.of(index, docLevelRules), detector, refreshPolicy, Monitor.NO_ID, Method.POST)); + for(Entry>> rulesPerCategory: docLevelRulesByCategory.entrySet()) { + String docLevelMonitorIdForCategory = detector.getDocLevelMonitorIdForRuleCategory(rulesPerCategory.getKey()); + List> rules = rulesPerCategory.getValue(); + + if(docLevelMonitorIdForCategory == null) { + monitorsToBeAdded.addAll(createDocLevelMonitorRequests(index, rules, detector, refreshPolicy, Monitor.NO_ID, Method.POST)); } else { - monitorsToBeUpdated.add(createDocLevelMonitorRequest(Pair.of(index, docLevelRules), detector, refreshPolicy, detector.getDocLevelMonitorId(), Method.PUT)); + monitorsToBeUpdated.addAll(createDocLevelMonitorRequests(index, rules, detector, refreshPolicy, docLevelMonitorIdForCategory, Method.PUT)); } } + // Both bucket and doc level monitors can be deleted + List monitorIdsToBeDeleted = detector.getMonitorIdsToBeDeleted(monitorsToBeUpdated); + List docRuleIndicies = detector.getRuleTopicsForMonitors(monitorIdsToBeDeleted); - List monitorIdsToBeDeleted = detector.getRuleIdMonitorIdMap().values().stream().collect(Collectors.toList()); - monitorIdsToBeDeleted.removeAll(monitorsToBeUpdated.stream().map(IndexMonitorRequest::getMonitorId).collect( - Collectors.toList())); - - updateAlertingMonitors(monitorsToBeAdded, monitorsToBeUpdated, monitorIdsToBeDeleted, refreshPolicy, listener); + updateAlertingMonitors(monitorsToBeAdded, monitorsToBeUpdated, monitorIdsToBeDeleted, docRuleIndicies, refreshPolicy, listener); } + + /** * Update list of monitors for the given detector * Executed in a steps: @@ -359,6 +385,7 @@ private void updateAlertingMonitors( List monitorsToBeAdded, List monitorsToBeUpdated, List monitorsToBeDeleted, + List ruleIndices, RefreshPolicy refreshPolicy, ActionListener> listener ) { @@ -382,7 +409,7 @@ private void updateAlertingMonitors( } StepListener> deleteMonitorStep = new StepListener<>(); - deleteAlertingMonitors(monitorsToBeDeleted, refreshPolicy, deleteMonitorStep); + deleteAlertingMonitors(monitorsToBeDeleted, ruleIndices, refreshPolicy, deleteMonitorStep); // 3. Delete alerting monitors (rules that are not provided by the user) deleteMonitorStep.whenComplete(deleteMonitorResponses -> // Return list of all updated + newly added monitors @@ -395,55 +422,62 @@ private void updateAlertingMonitors( }, listener::onFailure); } - private IndexMonitorRequest createDocLevelMonitorRequest(Pair>> logIndexToQueries, Detector detector, WriteRequest.RefreshPolicy refreshPolicy, String monitorId, RestRequest.Method restMethod) { - List docLevelMonitorInputs = new ArrayList<>(); + private List createDocLevelMonitorRequests(String index, List> logIndexToQueries, Detector detector, WriteRequest.RefreshPolicy refreshPolicy, String monitorId, RestRequest.Method restMethod) { + Map>> rulesByCategory = logIndexToQueries.stream().collect(Collectors.groupingBy(stringRulePair -> stringRulePair.getRight().getCategory())); + List requests = new ArrayList<>(); - List docLevelQueries = new ArrayList<>(); + for(Entry>> entry: rulesByCategory.entrySet()) { - for (Pair query: logIndexToQueries.getRight()) { - String id = query.getLeft(); + List docLevelMonitorInputs = new ArrayList<>(); + List docLevelQueries = new ArrayList<>(); - Rule rule = query.getRight(); - String name = query.getLeft(); + for (Pair query: entry.getValue()) { + String id = query.getLeft(); - String actualQuery = rule.getQueries().get(0).getValue(); + Rule rule = query.getRight(); + String name = query.getLeft(); - List tags = new ArrayList<>(); - tags.add(rule.getLevel()); - tags.add(rule.getCategory()); - tags.addAll(rule.getTags().stream().map(Value::getValue).collect(Collectors.toList())); + String actualQuery = rule.getQueries().get(0).getValue(); - DocLevelQuery docLevelQuery = new DocLevelQuery(id, name, actualQuery, tags); - docLevelQueries.add(docLevelQuery); - } - DocLevelMonitorInput docLevelMonitorInput = new DocLevelMonitorInput(detector.getName(), List.of(logIndexToQueries.getKey()), docLevelQueries); - docLevelMonitorInputs.add(docLevelMonitorInput); + List tags = new ArrayList<>(); + tags.add(rule.getLevel()); + tags.add(rule.getCategory()); + tags.addAll(rule.getTags().stream().map(Value::getValue).collect(Collectors.toList())); - List triggers = new ArrayList<>(); - List detectorTriggers = detector.getTriggers(); + DocLevelQuery docLevelQuery = new DocLevelQuery(id, name, actualQuery, tags); + docLevelQueries.add(docLevelQuery); + } + docLevelMonitorInputs.add(new DocLevelMonitorInput(detector.getName(), List.of(index), docLevelQueries)); - for (DetectorTrigger detectorTrigger: detectorTriggers) { - String id = detectorTrigger.getId(); - String name = detectorTrigger.getName(); - String severity = detectorTrigger.getSeverity(); - List actions = detectorTrigger.getActions(); - Script condition = detectorTrigger.convertToCondition(); + List triggers = new ArrayList<>(); + List detectorTriggers = detector.getTriggers(); - triggers.add(new DocumentLevelTrigger(id, name, severity, actions, condition)); - } + for (DetectorTrigger detectorTrigger: detectorTriggers) { + String id = detectorTrigger.getId(); + String name = detectorTrigger.getName(); + String severity = detectorTrigger.getSeverity(); + List actions = detectorTrigger.getActions(); + Script condition = detectorTrigger.convertToCondition(); - Monitor monitor = new Monitor(monitorId, Monitor.NO_VERSION, detector.getName(), detector.getEnabled(), detector.getSchedule(), detector.getLastUpdateTime(), detector.getEnabledTime(), - Monitor.MonitorType.DOC_LEVEL_MONITOR, detector.getUser(), 1, docLevelMonitorInputs, triggers, Map.of(), - new DataSources(detector.getRuleIndex(), - detector.getFindingsIndex(), - detector.getFindingsIndexPattern(), - detector.getAlertsIndex(), - detector.getAlertsHistoryIndex(), - detector.getAlertsHistoryIndexPattern(), - DetectorMonitorConfig.getRuleIndexMappingsByType(detector.getDetectorType()), - true), PLUGIN_OWNER_FIELD); + triggers.add(new DocumentLevelTrigger(id, name, severity, actions, condition)); + } - return new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null); + String category = entry.getKey(); + + Monitor monitor = new Monitor(monitorId, Monitor.NO_VERSION, detector.getName(), detector.getEnabled(), detector.getSchedule(), detector.getLastUpdateTime(), detector.getEnabledTime(), + Monitor.MonitorType.DOC_LEVEL_MONITOR, detector.getUser(), 1, docLevelMonitorInputs, triggers, Map.of(), + new DataSources(DetectorMonitorConfig.getRuleIndex(category), + DetectorMonitorConfig.getFindingsIndex(category), + DetectorMonitorConfig.getFindingsIndexPattern(category), + DetectorMonitorConfig.getAlertsIndex(category), + DetectorMonitorConfig.getAlertsHistoryIndex(category), + DetectorMonitorConfig.getAlertsHistoryIndexPattern(category), + DetectorMonitorConfig.getRuleIndexMappingsByType(), + true), PLUGIN_OWNER_FIELD); + + requests.add(new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null)); + } + return requests; } private List buildBucketLevelMonitorRequests(Pair>> logIndexToQueries, Detector detector, WriteRequest.RefreshPolicy refreshPolicy, String monitorId, RestRequest.Method restMethod) throws IOException, SigmaError { @@ -547,15 +581,17 @@ private IndexMonitorRequest createBucketLevelMonitorRequest( triggers.add(bucketLevelTrigger1); } **/ + String ruleCategory = rule.getCategory(); + Monitor monitor = new Monitor(monitorId, Monitor.NO_VERSION, detector.getName(), detector.getEnabled(), detector.getSchedule(), detector.getLastUpdateTime(), detector.getEnabledTime(), MonitorType.BUCKET_LEVEL_MONITOR, detector.getUser(), 1, bucketLevelMonitorInputs, triggers, Map.of(), - new DataSources(detector.getRuleIndex(), - detector.getFindingsIndex(), - detector.getFindingsIndexPattern(), - detector.getAlertsIndex(), - detector.getAlertsHistoryIndex(), - detector.getAlertsHistoryIndexPattern(), - DetectorMonitorConfig.getRuleIndexMappingsByType(detector.getDetectorType()), + new DataSources(DetectorMonitorConfig.getRuleIndex(ruleCategory), + DetectorMonitorConfig.getFindingsIndex(ruleCategory), + DetectorMonitorConfig.getFindingsIndexPattern(ruleCategory), + DetectorMonitorConfig.getAlertsIndex(ruleCategory), + DetectorMonitorConfig.getAlertsHistoryIndex(ruleCategory), + DetectorMonitorConfig.getAlertsHistoryIndexPattern(ruleCategory), + DetectorMonitorConfig.getRuleIndexMappingsByType(), true), PLUGIN_OWNER_FIELD); return new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null); @@ -596,15 +632,17 @@ public void onFailure(Exception e) { /** * Deletes the alerting monitors based on the given ids and notifies the listener that will be notified once all monitors have been deleted + * Beside deleting the monitors, checks and deletes the rule indices if they are empty * @param monitorIds monitor ids to be deleted * @param refreshPolicy * @param listener listener that will be notified once all the monitors are being deleted */ - private void deleteAlertingMonitors(List monitorIds, WriteRequest.RefreshPolicy refreshPolicy, ActionListener> listener){ + private void deleteAlertingMonitors(List monitorIds, List ruleIndices, WriteRequest.RefreshPolicy refreshPolicy, ActionListener> listener){ if (monitorIds == null || monitorIds.isEmpty()) { listener.onResponse(new ArrayList<>()); return; } + ActionListener deletesListener = new GroupedActionListener<>(new ActionListener<>() { @Override public void onResponse(Collection responses) { @@ -734,14 +772,9 @@ void prepareDetectorIndexing() throws IOException { void createDetector() { Detector detector = request.getDetector(); - String ruleTopic = detector.getDetectorType(); - request.getDetector().setAlertsIndex(DetectorMonitorConfig.getAlertsIndex(ruleTopic)); - request.getDetector().setAlertsHistoryIndex(DetectorMonitorConfig.getAlertsHistoryIndex(ruleTopic)); - request.getDetector().setAlertsHistoryIndexPattern(DetectorMonitorConfig.getAlertsHistoryIndexPattern(ruleTopic)); - request.getDetector().setFindingsIndex(DetectorMonitorConfig.getFindingsIndex(ruleTopic)); - request.getDetector().setFindingsIndexPattern(DetectorMonitorConfig.getFindingsIndexPattern(ruleTopic)); - request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndex(ruleTopic)); + // Refactored to support multiple log types + List ruleIndices = detector.getRuleIndices(); User originalContextUser = this.user; log.debug("user from original context is {}", originalContextUser); @@ -757,22 +790,23 @@ public void onResponse(AcknowledgedResponse acknowledgedResponse) { initRuleIndexAndImportRules(request, new ActionListener<>() { @Override public void onResponse(List monitorResponses) { + Pair, Map> monitorPairs = mapMonitorIds(monitorResponses); + request.getDetector().setMonitorIds(getMonitorIds(monitorResponses)); - request.getDetector().setRuleIdMonitorIdMap(mapMonitorIds(monitorResponses)); + request.getDetector().setDocLevelMonitorPerCategory(monitorPairs.getLeft()); + request.getDetector().setBucketRuleIdMonitorIdMap(monitorPairs.getRight()); try { indexDetector(); } catch (IOException e) { onFailures(e); } } - @Override public void onFailure(Exception e) { onFailures(e); } }); } - @Override public void onFailure(Exception e) { onFailures(e); @@ -838,21 +872,15 @@ void onGetResponse(Detector currentDetector, User user) { request.getDetector().setEnabledTime(currentDetector.getEnabledTime()); } request.getDetector().setMonitorIds(currentDetector.getMonitorIds()); - request.getDetector().setRuleIdMonitorIdMap(currentDetector.getRuleIdMonitorIdMap()); - Detector detector = request.getDetector(); + request.getDetector().setDocLevelMonitorPerCategory(currentDetector.getDocLevelMonitorPerCategory()); + request.getDetector().setBucketRuleIdMonitorIdMap(currentDetector.getBucketRuleIdMonitorIdMap()); - String ruleTopic = detector.getDetectorType(); + Detector detector = request.getDetector(); log.debug("user in update detector {}", user); - - request.getDetector().setAlertsIndex(DetectorMonitorConfig.getAlertsIndex(ruleTopic)); - request.getDetector().setAlertsHistoryIndex(DetectorMonitorConfig.getAlertsHistoryIndex(ruleTopic)); - request.getDetector().setAlertsHistoryIndexPattern(DetectorMonitorConfig.getAlertsHistoryIndexPattern(ruleTopic)); - request.getDetector().setFindingsIndex(DetectorMonitorConfig.getFindingsIndex(ruleTopic)); - request.getDetector().setFindingsIndexPattern(DetectorMonitorConfig.getFindingsIndexPattern(ruleTopic)); - request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndex(ruleTopic)); request.getDetector().setUser(user); + List ruleIndices = detector.getRuleIndices(); if (!detector.getInputs().isEmpty()) { try { @@ -862,8 +890,10 @@ public void onResponse(AcknowledgedResponse acknowledgedResponse) { initRuleIndexAndImportRules(request, new ActionListener<>() { @Override public void onResponse(List monitorResponses) { + Pair, Map> monitorPairs = mapMonitorIds(monitorResponses); request.getDetector().setMonitorIds(getMonitorIds(monitorResponses)); - request.getDetector().setRuleIdMonitorIdMap(mapMonitorIds(monitorResponses)); + request.getDetector().setDocLevelMonitorPerCategory(monitorPairs.getLeft()); + request.getDetector().setBucketRuleIdMonitorIdMap(monitorPairs.getRight()); try { indexDetector(); } catch (IOException e) { @@ -996,21 +1026,27 @@ public void onFailure(Exception e) { @SuppressWarnings("unchecked") public void importRules(IndexDetectorRequest request, ActionListener> listener) { final Detector detector = request.getDetector(); - final String ruleTopic = detector.getDetectorType(); + final DetectorInput detectorInput = detector.getInputs().get(0); final String logIndex = detectorInput.getIndices().get(0); + // Introduced breaking change - all detector types will be stored in detectorInput + final List detectorTypes = detector.getInputs().get(0).getDetectorTypes(); List ruleIds = detectorInput.getPrePackagedRules().stream().map(DetectorRule::getId).collect(Collectors.toList()); - - QueryBuilder queryBuilder = - QueryBuilders.nestedQuery("rule", - QueryBuilders.boolQuery().must( - QueryBuilders.matchQuery("rule.category", ruleTopic) - ).must( - QueryBuilders.termsQuery("_id", ruleIds.toArray(new String[]{})) - ), - ScoreMode.Avg - ); + QueryBuilder queryBuilder; + + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + for(DetectorType detectorType: detectorTypes) { + boolQueryBuilder = boolQueryBuilder.should(QueryBuilders.nestedQuery("rule", + QueryBuilders.boolQuery().must( + QueryBuilders.matchQuery("rule.category", detectorType.getDetectorType()) + ).must( + QueryBuilders.termsQuery("_id", ruleIds.toArray(new String[]{})) + ), + ScoreMode.Avg + )); + } + queryBuilder = boolQueryBuilder; SearchRequest searchRequest = new SearchRequest(Rule.PRE_PACKAGED_RULES_INDEX) .source(new SearchSourceBuilder() @@ -1184,20 +1220,23 @@ private List getMonitorIds(List monitorResponses) * @param monitorResponses index monitor responses * @return map of monitor ids */ - private Map mapMonitorIds(List monitorResponses) { - return monitorResponses.stream().collect( - Collectors.toMap( - // In the case of bucket level monitors rule id is trigger id - it -> { + private Pair, Map> mapMonitorIds(List monitorResponses) { + Map bucketMonitorIdPerRuleId = new HashMap<>(); + Map docLevelMonitorIdPerCategory = new HashMap<>(); + monitorResponses.stream().forEach(it -> { + // In the case of bucket level monitors rule id is trigger id if (MonitorType.BUCKET_LEVEL_MONITOR == it.getMonitor().getMonitorType()) { - return it.getMonitor().getTriggers().get(0).getId(); - } else { - return Detector.DOC_LEVEL_MONITOR; + bucketMonitorIdPerRuleId.put(it.getMonitor().getTriggers().get(0).getId(), it.getId()); } - }, - IndexMonitorResponse::getId - ) + // TODO - think something better?; In the case of doc level monitors, key is the rule category/ detector type + // test_windows : abc-xyz-asd + else { + docLevelMonitorIdPerCategory.put(DetectorMonitorConfig.getRuleCategoryFromFindingIndexName(it.getMonitor().getDataSources().getFindingsIndex()), it.getId()); + } + } ); + + return Pair.of(docLevelMonitorIdPerCategory, bucketMonitorIdPerRuleId); } } diff --git a/src/main/resources/mappings/detectors.json b/src/main/resources/mappings/detectors.json index aefa0dbf6..c8402654d 100644 --- a/src/main/resources/mappings/detectors.json +++ b/src/main/resources/mappings/detectors.json @@ -160,6 +160,14 @@ "type": "text" } } + }, + "detector_types": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } } } } diff --git a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java index a4f2df778..1588c1a28 100644 --- a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java +++ b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java @@ -4,6 +4,7 @@ */ package org.opensearch.securityanalytics; +import java.io.UnsupportedEncodingException; import java.util.Set; import org.apache.http.HttpHost; import java.util.ArrayList; @@ -80,7 +81,6 @@ import static org.opensearch.securityanalytics.util.RuleTopicIndices.ruleTopicIndexSettings; public class SecurityAnalyticsRestTestCase extends OpenSearchRestTestCase { - protected void createRuleTopicIndex(String detectorType, String additionalMapping) throws IOException { String mappings = "" + @@ -311,6 +311,33 @@ protected List getRandomPrePackagedRules() throws IOException { return hits.stream().map(hit -> hit.get("_id").toString()).collect(Collectors.toList()); } + protected List getRandomPrePackagedRules(String ruleCategory) throws IOException { + String request = "{\n" + + " \"from\": 0\n," + + " \"size\": 2000\n," + + " \"query\": {\n" + + " \"nested\": {\n" + + " \"path\": \"rule\",\n" + + " \"query\": {\n" + + " \"bool\": {\n" + + " \"must\": [\n" + + " { \"match\": {\"rule.category\": \"" + ruleCategory + "\"}}\n" + + " ]\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; + + Response searchResponse = makeRequest(client(), "POST", String.format(Locale.getDefault(), "%s/_search", SecurityAnalyticsPlugin.RULE_BASE_URI), Collections.singletonMap("pre_packaged", "true"), + new StringEntity(request), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Searching rules failed", RestStatus.OK, restStatus(searchResponse)); + + Map responseBody = asMap(searchResponse); + List> hits = ((List>) ((Map) responseBody.get("hits")).get("hits")); + return hits.stream().map(hit -> hit.get("_id").toString()).collect(Collectors.toList()); + } + protected List createAggregationRules () throws IOException { return new ArrayList<>(Arrays.asList(createRule(productIndexAvgAggRule()), createRule(sumAggregationTestRule()))); } @@ -323,6 +350,14 @@ protected String createRule(String rule) throws IOException { return responseBody.get("_id").toString(); } + protected String createRule(String rule, String ruleCategory) throws IOException { + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", ruleCategory), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + Map responseBody = asMap(createResponse); + return responseBody.get("_id").toString(); + } + protected List getPrePackagedRules(String ruleCategory) throws IOException { String request = "{\n" + " \"from\": 0\n," + diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 34afda245..ce556c73c 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -5,6 +5,7 @@ package org.opensearch.securityanalytics; import com.carrotsearch.randomizedtesting.generators.RandomNumbers; +import java.util.Arrays; import org.apache.lucene.tests.util.LuceneTestCase; import org.opensearch.common.bytes.BytesReference; import org.opensearch.common.xcontent.LoggingDeprecationHandler; @@ -22,6 +23,7 @@ import org.opensearch.script.Script; import org.opensearch.script.ScriptType; import org.opensearch.securityanalytics.model.Detector; +import org.opensearch.securityanalytics.model.Detector.DetectorType; import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.model.DetectorRule; import org.opensearch.securityanalytics.model.DetectorTrigger; @@ -49,36 +51,47 @@ static class AccessRoles { public static Detector randomDetector(List rules) { DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), Collections.emptyList(), - rules.stream().map(DetectorRule::new).collect(Collectors.toList())); - return randomDetector(null, null, null, List.of(input), List.of(), null, null, null, null); + rules.stream().map(DetectorRule::new).collect(Collectors.toList()), Collections.emptyList()); + return randomDetector(null, null, List.of(input), List.of(), null, null, null, null); + } + + public static Detector randomDetectorWithoutDetectorType(List rules) { + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), Collections.emptyList(), + rules.stream().map(DetectorRule::new).collect(Collectors.toList()), Collections.emptyList()); + Detector detector = randomDetector(null, null, List.of(input), List.of(), null, null, null, null); + detector.getInputs().get(0).setDetectorTypes(Collections.emptyList()); + return detector; } public static Detector randomDetectorWithInputs(List inputs) { - return randomDetector(null, null, null, inputs, List.of(), null, null, null, null); + return randomDetector(null, null, inputs, List.of(), null, null, null, null); } public static Detector randomDetectorWithTriggers(List triggers) { - return randomDetector(null, null, null, List.of(), triggers, null, null, null, null); + return randomDetector(null, null, List.of(), triggers, null, null, null, null); } public static Detector randomDetectorWithTriggers(List rules, List triggers) { DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), Collections.emptyList(), - rules.stream().map(DetectorRule::new).collect(Collectors.toList())); - return randomDetector(null, null, null, List.of(input), triggers, null, null, null, null); + rules.stream().map(DetectorRule::new).collect(Collectors.toList()), new ArrayList<>()); + return randomDetector(null, null, List.of(input), triggers, null, null, null, null); + } + + public static Detector randomDetectorWithTriggers(DetectorInput input, List triggers) { + return randomDetector(null, null, List.of(input), triggers, null, null, null, null); } public static Detector randomDetectorWithTriggers(List rules, List triggers, List inputIndices) { DetectorInput input = new DetectorInput("windows detector for security analytics", inputIndices, Collections.emptyList(), - rules.stream().map(DetectorRule::new).collect(Collectors.toList())); - return randomDetector(null, null, null, List.of(input), triggers, null, null, null, null); + rules.stream().map(DetectorRule::new).collect(Collectors.toList()), Collections.emptyList()); + return randomDetector(null, null, List.of(input), triggers, null, null, null, null); } public static Detector randomDetectorWithInputsAndTriggers(List inputs, List triggers) { - return randomDetector(null, null, null, inputs, triggers, null, null, null, null); + return randomDetector(null, null, inputs, triggers, null, null, null, null); } - public static Detector randomDetectorWithTriggers(List rules, List triggers, Detector.DetectorType detectorType, DetectorInput input) { - return randomDetector(null, detectorType, null, List.of(input), triggers, null, null, null, null); + public static Detector randomDetectorWithTriggers(List rules, List triggers, DetectorInput input) { + return randomDetector(null, null, List.of(input), triggers, null, null, null, null); } public static Detector randomDetector(String name, - Detector.DetectorType detectorType, User user, List inputs, List triggers, @@ -89,15 +102,15 @@ public static Detector randomDetector(String name, if (name == null) { name = OpenSearchRestTestCase.randomAlphaOfLength(10); } - if (detectorType == null) { - detectorType = Detector.DetectorType.valueOf(randomDetectorType().toUpperCase(Locale.ROOT)); - } if (user == null) { user = randomUser(); } if (inputs == null) { inputs = Collections.emptyList(); } + if (inputs != null && inputs.get(0).getDetectorTypes().isEmpty()) { + inputs.get(0).setDetectorTypes(Arrays.asList(Detector.DetectorType.valueOf(randomDetectorType().toUpperCase(Locale.ROOT)))); + } if (schedule == null) { schedule = new IntervalSchedule(5, ChronoUnit.MINUTES, null); } @@ -115,7 +128,7 @@ public static Detector randomDetector(String name, if (inputs.size() == 0) { inputs = new ArrayList<>(); - DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), Collections.emptyList(), null); + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), Collections.emptyList(), null, new ArrayList<>()); inputs.add(input); } if (triggers.size() == 0) { @@ -124,19 +137,19 @@ public static Detector randomDetector(String name, DetectorTrigger trigger = new DetectorTrigger(null, "windows-trigger", "1", List.of(randomDetectorType()), List.of("QuarksPwDump Clearing Access History"), List.of("high"), List.of("T0008"), List.of()); triggers.add(trigger); } - return new Detector(null, null, name, enabled, schedule, lastUpdateTime, enabledTime, detectorType, user, inputs, triggers, Collections.singletonList(""), "", "", "", "", "", "", Collections.emptyMap()); + return new Detector(null, null, name, enabled, schedule, lastUpdateTime, enabledTime, user, inputs, triggers, Collections.singletonList(""), "", "", "", "", "", "", Collections.emptyMap(), Collections.emptyMap()); } public static Detector randomDetectorWithNoUser() { String name = OpenSearchRestTestCase.randomAlphaOfLength(10); Detector.DetectorType detectorType = Detector.DetectorType.valueOf(randomDetectorType().toUpperCase(Locale.ROOT)); - List inputs = Collections.emptyList(); + List inputs = List.of(new DetectorInput("", Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), List.of(detectorType))); Schedule schedule = new IntervalSchedule(5, ChronoUnit.MINUTES, null); Boolean enabled = OpenSearchTestCase.randomBoolean(); Instant enabledTime = enabled ? Instant.now().truncatedTo(ChronoUnit.MILLIS) : null; Instant lastUpdateTime = Instant.now().truncatedTo(ChronoUnit.MILLIS); - return new Detector(null, null, name, enabled, schedule, lastUpdateTime, enabledTime, detectorType, null, inputs, Collections.emptyList(),Collections.singletonList(""), "", "", "", "", "", "", Collections.emptyMap()); + return new Detector(null, null, name, enabled, schedule, lastUpdateTime, enabledTime, null, inputs, Collections.emptyList(),Collections.singletonList(""), "", "", "", "", "", "", Collections.emptyMap(), Collections.emptyMap()); } public static String randomRule() { @@ -323,6 +336,10 @@ public static String randomDetectorType() { return "test_windows"; } + public static String windowsDetectorType() { + return "windows"; + } + public static DetectorInput randomDetectorInput() { String description = OpenSearchRestTestCase.randomAlphaOfLength(randomInt(10)); @@ -336,7 +353,7 @@ public static DetectorInput randomDetectorInput() { detectorRules.add(randomDetectorRule()); } - return new DetectorInput(description, indices, detectorRules, detectorRules); + return new DetectorInput(description, indices, detectorRules, detectorRules, Collections.emptyList()); } public static DetectorRule randomDetectorRule() { diff --git a/src/test/java/org/opensearch/securityanalytics/action/IndexDetectorResponseTests.java b/src/test/java/org/opensearch/securityanalytics/action/IndexDetectorResponseTests.java index ad6a110e2..e26ecface 100644 --- a/src/test/java/org/opensearch/securityanalytics/action/IndexDetectorResponseTests.java +++ b/src/test/java/org/opensearch/securityanalytics/action/IndexDetectorResponseTests.java @@ -11,6 +11,7 @@ import org.opensearch.rest.RestStatus; import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig; import org.opensearch.securityanalytics.model.Detector; +import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; @@ -29,8 +30,6 @@ public void testIndexDetectorPostResponse() throws IOException { CronSchedule cronSchedule = new CronSchedule(cronExpression, ZoneId.of("Asia/Kolkata"), testInstance); - Detector.DetectorType detectorType = Detector.DetectorType.LINUX; - String detectorTypeString = detectorType.getDetectorType(); Detector detector = new Detector( "123", 0L, @@ -39,9 +38,8 @@ public void testIndexDetectorPostResponse() throws IOException { cronSchedule, Instant.now(), Instant.now(), - detectorType, randomUser(), - List.of(), + List.of(new DetectorInput("", Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), List.of(Detector.DetectorType.LINUX))), List.of(), List.of("1", "2", "3"), DetectorMonitorConfig.getRuleIndex(Detector.DetectorType.OTHERS_APPLICATION.getDetectorType()), @@ -50,6 +48,7 @@ public void testIndexDetectorPostResponse() throws IOException { null, null, DetectorMonitorConfig.getFindingsIndex(Detector.DetectorType.OTHERS_APPLICATION.getDetectorType()), + Collections.emptyMap(), Collections.emptyMap() ); IndexDetectorResponse response = new IndexDetectorResponse("1234", 1L, RestStatus.OK, detector); diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java b/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java index b6df74548..93ce771dc 100644 --- a/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java +++ b/src/test/java/org/opensearch/securityanalytics/alerts/AlertingServiceTests.java @@ -27,6 +27,7 @@ import org.opensearch.securityanalytics.action.GetDetectorResponse; import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig; import org.opensearch.securityanalytics.model.Detector; +import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.transport.TransportIndexDetectorAction; import org.opensearch.test.OpenSearchTestCase; @@ -53,9 +54,8 @@ public void testGetAlerts_success() { new CronSchedule("31 * * * *", ZoneId.of("Asia/Kolkata"), Instant.ofEpochSecond(1538164858L)), Instant.now(), Instant.now(), - Detector.DetectorType.OTHERS_APPLICATION, null, - List.of(), + List.of(new DetectorInput("", Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), List.of( Detector.DetectorType.OTHERS_APPLICATION))), List.of(), List.of("monitor_id1", "monitor_id2"), DetectorMonitorConfig.getRuleIndex(Detector.DetectorType.OTHERS_APPLICATION.getDetectorType()), @@ -64,6 +64,7 @@ public void testGetAlerts_success() { null, null, DetectorMonitorConfig.getFindingsIndex(Detector.DetectorType.OTHERS_APPLICATION.getDetectorType()), + Collections.emptyMap(), Collections.emptyMap() ); GetDetectorResponse getDetectorResponse = new GetDetectorResponse("detector_id123", 1L, RestStatus.OK, detector); @@ -225,9 +226,8 @@ public void testGetFindings_getFindingsByMonitorIdFailures() { new CronSchedule("31 * * * *", ZoneId.of("Asia/Kolkata"), Instant.ofEpochSecond(1538164858L)), Instant.now(), Instant.now(), - Detector.DetectorType.OTHERS_APPLICATION, null, - List.of(), + List.of(new DetectorInput("", Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), List.of(Detector.DetectorType.OTHERS_APPLICATION))), List.of(), List.of("monitor_id1", "monitor_id2"), DetectorMonitorConfig.getRuleIndex(Detector.DetectorType.OTHERS_APPLICATION.getDetectorType()), @@ -236,6 +236,7 @@ public void testGetFindings_getFindingsByMonitorIdFailures() { null, null, DetectorMonitorConfig.getFindingsIndex(Detector.DetectorType.OTHERS_APPLICATION.getDetectorType()), + Collections.emptyMap(), Collections.emptyMap() ); GetDetectorResponse getDetectorResponse = new GetDetectorResponse("detector_id123", 1L, RestStatus.OK, detector); diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java index b373b8211..6414eb0c1 100644 --- a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java +++ b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java @@ -12,14 +12,18 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.apache.http.HttpStatus; import org.apache.http.entity.StringEntity; import org.apache.http.message.BasicHeader; import org.junit.Assert; import org.opensearch.client.Request; +import org.opensearch.client.Requests; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; +import org.opensearch.cluster.health.ClusterHealthStatus; +import org.opensearch.commons.alerting.model.Monitor.MonitorType; import org.opensearch.commons.alerting.model.action.Action; import org.opensearch.rest.RestStatus; import org.opensearch.search.SearchHit; @@ -28,18 +32,21 @@ import org.opensearch.securityanalytics.action.AlertDto; import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig; import org.opensearch.securityanalytics.model.Detector; +import org.opensearch.securityanalytics.model.Detector.DetectorType; import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.model.DetectorRule; import org.opensearch.securityanalytics.model.DetectorTrigger; import static org.opensearch.securityanalytics.TestHelpers.netFlowMappings; import static org.opensearch.securityanalytics.TestHelpers.randomAction; +import static org.opensearch.securityanalytics.TestHelpers.randomAggregationRule; import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType; import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputsAndTriggers; import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithTriggers; import static org.opensearch.securityanalytics.TestHelpers.randomDoc; import static org.opensearch.securityanalytics.TestHelpers.randomIndex; import static org.opensearch.securityanalytics.TestHelpers.randomRule; +import static org.opensearch.securityanalytics.TestHelpers.windowsDetectorType; import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_INDEX_MAX_AGE; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_MAX_DOCS; @@ -55,7 +62,7 @@ public void testGetAlerts_success() throws IOException { String rule = randomRule(); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", randomDetectorType()), - new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); @@ -66,10 +73,10 @@ public void testGetAlerts_success() throws IOException { Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); // both req params and req body are supported createMappingRequest.setJsonEntity( - "{ \"index_name\":\"" + index + "\"," + - " \"rule_topic\":\"" + randomDetectorType() + "\", " + - " \"partial\":true" + - "}" + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" ); Response response = client().performRequest(createMappingRequest); @@ -79,8 +86,8 @@ public void testGetAlerts_success() throws IOException { Action triggerAction = randomAction(createDestination()); Detector detector = randomDetectorWithInputsAndTriggers(List.of(new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(createdId)), - getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList()))), - List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(), List.of(createdId), List.of(), List.of("attack.defense_evasion"), List.of(triggerAction)))); + getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList()), new ArrayList<>())), + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(), List.of(createdId), List.of(), List.of("attack.defense_evasion"), List.of(triggerAction)))); createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); @@ -90,12 +97,12 @@ public void testGetAlerts_success() throws IOException { createdId = responseBody.get("_id").toString(); String request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + createdId + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; List hits = executeSearch(Detector.DETECTORS_INDEX, request); SearchHit hit = hits.get(0); @@ -120,7 +127,7 @@ public void testGetAlerts_success() throws IOException { for (Map.Entry actionResult: actionResults.entrySet()) { Map actionOutput = ((Map>) actionResult.getValue()).get("output"); String expectedMessage = triggerAction.getSubjectTemplate().getIdOrCode().replace("{{ctx.detector.name}}", detector.getName()) - .replace("{{ctx.trigger.name}}", "test-trigger").replace("{{ctx.trigger.severity}}", "1"); + .replace("{{ctx.trigger.name}}", "test-trigger").replace("{{ctx.trigger.severity}}", "1"); Assert.assertEquals(expectedMessage, actionOutput.get("subject")); Assert.assertEquals(expectedMessage, actionOutput.get("message")); @@ -129,11 +136,11 @@ public void testGetAlerts_success() throws IOException { } request = "{\n" + - " \"query\" : {\n" + - " \"match_all\":{\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; hits = new ArrayList<>(); while (hits.size() == 0) { @@ -152,10 +159,10 @@ public void testGetAlerts_success() throws IOException { params = new HashMap<>(); String body = String.format(Locale.getDefault(), "{\"alerts\":[\"%s\"]}", alertId); Request post = new Request("POST", String.format( - Locale.getDefault(), - "%s/%s/_acknowledge/alerts", - SecurityAnalyticsPlugin.DETECTOR_BASE_URI, - detectorId)); + Locale.getDefault(), + "%s/%s/_acknowledge/alerts", + SecurityAnalyticsPlugin.DETECTOR_BASE_URI, + detectorId)); post.setJsonEntity(body); Response ackAlertsResponse = client().performRequest(post); assertNotNull(ackAlertsResponse); @@ -166,7 +173,7 @@ public void testGetAlerts_success() throws IOException { } public void testGetAlerts_noDetector_failure() throws IOException { - // Call GetAlerts API + // Call GetAlerts API Map params = new HashMap<>(); params.put("detector_id", "nonexistent_detector_id"); try { @@ -184,10 +191,10 @@ public void testAckAlerts_WithInvalidDetectorAlertsCombination() throws IOExcept Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); // both req params and req body are supported createMappingRequest.setJsonEntity( - "{ \"index_name\":\"" + index + "\"," + - " \"rule_topic\":\"" + randomDetectorType() + "\", " + - " \"partial\":true" + - "}" + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" ); Response response = client().performRequest(createMappingRequest); @@ -197,14 +204,14 @@ public void testAckAlerts_WithInvalidDetectorAlertsCombination() throws IOExcept Action triggerAction = randomAction(createDestination()); Detector detector = randomDetectorWithInputsAndTriggers(List.of(new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(), - getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList()))), - List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(), List.of(), List.of(), List.of("attack.defense_evasion"), List.of(triggerAction)))); + getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList()), new ArrayList<>())), + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(), List.of(), List.of(), List.of("attack.defense_evasion"), List.of(triggerAction)))); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); Detector detector1 = randomDetectorWithInputsAndTriggers(List.of(new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(), - getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList()))), - List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(), List.of(), List.of(), List.of("attack.defense_evasion"), List.of(triggerAction)))); + getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList()), new ArrayList<>())), + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(), List.of(), List.of(), List.of("attack.defense_evasion"), List.of(triggerAction)))); Response createResponse1 = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector1)); Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); @@ -215,12 +222,12 @@ public void testAckAlerts_WithInvalidDetectorAlertsCombination() throws IOExcept String id1 = asMap(createResponse1).get("_id").toString(); String request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + createdId + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; List hits = executeSearch(Detector.DETECTORS_INDEX, request); SearchHit hit = hits.get(0); @@ -238,11 +245,11 @@ public void testAckAlerts_WithInvalidDetectorAlertsCombination() throws IOExcept Assert.assertEquals(5, noOfSigmaRuleMatches); request = "{\n" + - " \"query\" : {\n" + - " \"match_all\":{\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; hits = new ArrayList<>(); while (hits.size() == 0) { @@ -260,10 +267,10 @@ public void testAckAlerts_WithInvalidDetectorAlertsCombination() throws IOExcept params = new HashMap<>(); String body = String.format(Locale.getDefault(), "{\"alerts\":[\"%s\"]}", alertId); Request post = new Request("POST", String.format( - Locale.getDefault(), - "%s/%s/_acknowledge/alerts", - SecurityAnalyticsPlugin.DETECTOR_BASE_URI, - id1)); + Locale.getDefault(), + "%s/%s/_acknowledge/alerts", + SecurityAnalyticsPlugin.DETECTOR_BASE_URI, + id1)); post.setJsonEntity(body); try { @@ -280,7 +287,7 @@ public void testAckAlertsWithInvalidDetector() throws IOException { String rule = randomRule(); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", randomDetectorType()), - new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); Map responseBody = asMap(createResponse); @@ -291,10 +298,10 @@ public void testAckAlertsWithInvalidDetector() throws IOException { Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); // both req params and req body are supported createMappingRequest.setJsonEntity( - "{ \"index_name\":\"" + index + "\"," + - " \"rule_topic\":\"" + randomDetectorType() + "\", " + - " \"partial\":true" + - "}" + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" ); Response response = client().performRequest(createMappingRequest); @@ -304,8 +311,8 @@ public void testAckAlertsWithInvalidDetector() throws IOException { Action triggerAction = randomAction(createDestination()); Detector detector = randomDetectorWithInputsAndTriggers(List.of(new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(createdId)), - getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList()))), - List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(), List.of(createdId), List.of(), List.of("attack.defense_evasion"), List.of(triggerAction)))); + getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList()), Collections.emptyList())), + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(), List.of(createdId), List.of(), List.of("attack.defense_evasion"), List.of(triggerAction)))); createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); @@ -315,12 +322,12 @@ public void testAckAlertsWithInvalidDetector() throws IOException { createdId = responseBody.get("_id").toString(); String request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + createdId + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; List hits = executeSearch(Detector.DETECTORS_INDEX, request); SearchHit hit = hits.get(0); @@ -345,7 +352,7 @@ public void testAckAlertsWithInvalidDetector() throws IOException { for (Map.Entry actionResult: actionResults.entrySet()) { Map actionOutput = ((Map>) actionResult.getValue()).get("output"); String expectedMessage = triggerAction.getSubjectTemplate().getIdOrCode().replace("{{ctx.detector.name}}", detector.getName()) - .replace("{{ctx.trigger.name}}", "test-trigger").replace("{{ctx.trigger.severity}}", "1"); + .replace("{{ctx.trigger.name}}", "test-trigger").replace("{{ctx.trigger.severity}}", "1"); Assert.assertEquals(expectedMessage, actionOutput.get("subject")); Assert.assertEquals(expectedMessage, actionOutput.get("message")); @@ -354,11 +361,11 @@ public void testAckAlertsWithInvalidDetector() throws IOException { } request = "{\n" + - " \"query\" : {\n" + - " \"match_all\":{\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; hits = new ArrayList<>(); while (hits.size() == 0) { @@ -376,10 +383,10 @@ public void testAckAlertsWithInvalidDetector() throws IOException { String detectorId = (String) ((ArrayList>) getAlertsBody.get("alerts")).get(0).get("detector_id"); String body = String.format(Locale.getDefault(), "{\"alerts\":[\"%s\"]}", alertId); Request post = new Request("POST", String.format( - Locale.getDefault(), - "%s/%s/_acknowledge/alerts", - SecurityAnalyticsPlugin.DETECTOR_BASE_URI, - java.util.UUID.randomUUID())); + Locale.getDefault(), + "%s/%s/_acknowledge/alerts", + SecurityAnalyticsPlugin.DETECTOR_BASE_URI, + java.util.UUID.randomUUID())); post.setJsonEntity(body); try { @@ -390,10 +397,10 @@ public void testAckAlertsWithInvalidDetector() throws IOException { body = String.format(Locale.getDefault(), "{\"alerts\":[\"%s\"]}", java.util.UUID.randomUUID()); post = new Request("POST", String.format( - Locale.getDefault(), - "%s/%s/_acknowledge/alerts", - SecurityAnalyticsPlugin.DETECTOR_BASE_URI, - detectorId)); + Locale.getDefault(), + "%s/%s/_acknowledge/alerts", + SecurityAnalyticsPlugin.DETECTOR_BASE_URI, + detectorId)); post.setJsonEntity(body); } @@ -404,10 +411,10 @@ public void testGetAlerts_byDetectorType_success() throws IOException, Interrupt Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); // both req params and req body are supported createMappingRequest.setJsonEntity( - "{ \"index_name\":\"" + index + "\"," + - " \"rule_topic\":\"" + randomDetectorType() + "\", " + - " \"partial\":true" + - "}" + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" ); Response response = client().performRequest(createMappingRequest); @@ -423,12 +430,12 @@ public void testGetAlerts_byDetectorType_success() throws IOException, Interrupt String createdId = responseBody.get("_id").toString(); String request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + createdId + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; List hits = executeSearch(Detector.DETECTORS_INDEX, request); SearchHit hit = hits.get(0); @@ -444,11 +451,11 @@ public void testGetAlerts_byDetectorType_success() throws IOException, Interrupt Assert.assertEquals(5, noOfSigmaRuleMatches); request = "{\n" + - " \"query\" : {\n" + - " \"match_all\":{\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; hits = new ArrayList<>(); while (hits.size() == 0) { @@ -464,6 +471,244 @@ public void testGetAlerts_byDetectorType_success() throws IOException, Interrupt Assert.assertEquals(1, getAlertsBody.get("total_alerts")); } + public void testGetAlerts_byDetectorType_multipleDetectorTypes_success() throws IOException, InterruptedException { + String testOpCode = "Test"; + + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + windowsDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + List prepackagedRules = getRandomPrePackagedRules(); + String maxRuleId = createRule(randomAggregationRule("max", " > 3", testOpCode)); + String randomDocRuleId = createRule(randomRule(), "windows"); + + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(maxRuleId), new DetectorRule(randomDocRuleId)), + prepackagedRules.stream().map(DetectorRule::new).collect(Collectors.toList()), List.of(DetectorType.TEST_WINDOWS, DetectorType.WINDOWS)); + + Detector detector = randomDetectorWithTriggers(input, List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(windowsDetectorType(), randomDetectorType()), List.of(), List.of(), List.of(), List.of()))); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String createdId = responseBody.get("_id").toString(); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + Map detectorMap = (HashMap)(hit.getSourceAsMap().get("detector")); + List monitorIds = ((List) (detectorMap).get("monitor_id")); + assertEquals(3, monitorIds.size()); + + indexDoc(index, "1", randomDoc(5, 3, testOpCode)); + indexDoc(index, "2", randomDoc(2, 3, testOpCode)); + indexDoc(index, "3", randomDoc(4, 3, testOpCode)); + indexDoc(index, "4", randomDoc(6, 2, testOpCode)); + indexDoc(index, "5", randomDoc(1, 1, testOpCode)); + + client().performRequest(new Request("POST", "_refresh")); + Map numberOfMonitorTypes = new HashMap<>(); + + Map docLevelMonitorIdPerCategory = ((Map)((Map)hit.getSourceAsMap().get("detector")).get("doc_monitor_id_per_category")); + // Swapping keys and values + Map ruleIdRuleCategoryMap = docLevelMonitorIdPerCategory.entrySet().stream().collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)); + + for(String monitorId: monitorIds) { + Map monitor = (Map)(entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + monitorId)))).get("monitor"); + numberOfMonitorTypes.merge(monitor.get("monitor_type"), 1, Integer::sum); + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + if (MonitorType.DOC_LEVEL_MONITOR.getValue().equals(monitor.get("monitor_type"))) { + String ruleCategory = ruleIdRuleCategoryMap.get(monitorId); + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + if (ruleCategory.toLowerCase(Locale.ROOT).equals(DetectorType.WINDOWS.getDetectorType().toLowerCase(Locale.ROOT))) { + assertEquals(1, noOfSigmaRuleMatches); + } else if (ruleCategory.toLowerCase(Locale.ROOT).equals(DetectorType.TEST_WINDOWS.getDetectorType().toLowerCase(Locale.ROOT))){ + assertEquals(5, noOfSigmaRuleMatches); + } + } else { + List> buckets = ((List>)(((Map)((Map)((Map)((List)((Map) executeResults.get("input_results")).get("results")).get(0)).get("aggregations")).get("result_agg")).get("buckets"))); + Integer docCount = buckets.stream().mapToInt(it -> (Integer)it.get("doc_count")).sum(); + assertEquals(5, docCount.intValue()); + List triggerResultBucketKeys = ((Map)((Map) ((Map)executeResults.get("trigger_results")).get(maxRuleId)).get("agg_result_buckets")).keySet().stream().collect(Collectors.toList()); + assertEquals(List.of("2", "3"), triggerResultBucketKeys); + } + } + + request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + hits = new ArrayList<>(); + + while (hits.size() == 0) { + hits = executeSearch(DetectorMonitorConfig.getAlertsIndex("windows"), request); + } + // Call GetAlerts API + Map params = new HashMap<>(); + params.put("detectorType", "windows"); + Response getAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.ALERTS_BASE_URI, params, null); + Map getAlertsBody = asMap(getAlertsResponse); + + assertEquals(5, getAlertsBody.get("total_alerts")); + + hits = new ArrayList<>(); + while (hits.size() == 0) { + hits = executeSearch(DetectorMonitorConfig.getAlertsIndex("test_windows"), request); + } + // Call GetAlerts API + params.put("detectorType", "test_windows"); + getAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.ALERTS_BASE_URI, params, null); + getAlertsBody = asMap(getAlertsResponse); + // TODO enable asserts here when able + // 5 prepackaged rule matches for 5 documents + 1 custom doc rule + 1 custom bucket rule + Assert.assertEquals(7, getAlertsBody.get("total_alerts")); + } + + public void testGetAlerts_byDetectorId_multipleDetectorTypes_success() throws IOException, InterruptedException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + windowsDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + List prepackagedRules = getRandomPrePackagedRules(); + String testOpCode = "Test"; + String maxRuleId = createRule(randomAggregationRule("max", " > 3", testOpCode)); + String randomDocRuleId = createRule(randomRule(), "windows"); + + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(maxRuleId), new DetectorRule(randomDocRuleId)), + prepackagedRules.stream().map(DetectorRule::new).collect(Collectors.toList()), List.of(DetectorType.TEST_WINDOWS, DetectorType.WINDOWS)); + + Detector detector = randomDetectorWithTriggers(input, List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(windowsDetectorType(), randomDetectorType()), List.of(), List.of(), List.of(), List.of()))); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String detectorId = responseBody.get("_id").toString(); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + Map detectorMap = (HashMap)(hit.getSourceAsMap().get("detector")); + List monitorIds = ((List) (detectorMap).get("monitor_id")); + assertEquals(3, monitorIds.size()); + + indexDoc(index, "1", randomDoc(5, 3, testOpCode)); + indexDoc(index, "2", randomDoc(2, 3, testOpCode)); + indexDoc(index, "3", randomDoc(4, 3, testOpCode)); + indexDoc(index, "4", randomDoc(6, 2, testOpCode)); + indexDoc(index, "5", randomDoc(1, 1, testOpCode)); + + client().performRequest(new Request("POST", "_refresh")); + Map numberOfMonitorTypes = new HashMap<>(); + + Map docLevelMonitorIdPerCategory = ((Map)((Map)hit.getSourceAsMap().get("detector")).get("doc_monitor_id_per_category")); + // Swapping keys and values + Map ruleIdRuleCategoryMap = docLevelMonitorIdPerCategory.entrySet().stream().collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)); + + for(String monitorId: monitorIds) { + Map monitor = (Map)(entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + monitorId)))).get("monitor"); + numberOfMonitorTypes.merge(monitor.get("monitor_type"), 1, Integer::sum); + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + if (MonitorType.DOC_LEVEL_MONITOR.getValue().equals(monitor.get("monitor_type"))) { + String ruleCategory = ruleIdRuleCategoryMap.get(monitorId); + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + if (ruleCategory.toLowerCase(Locale.ROOT).equals(DetectorType.WINDOWS.getDetectorType().toLowerCase(Locale.ROOT))) { + assertEquals("Number of doc level rules for windows category not correct", 1, noOfSigmaRuleMatches); + } else if (ruleCategory.toLowerCase(Locale.ROOT).equals(DetectorType.TEST_WINDOWS.getDetectorType().toLowerCase(Locale.ROOT))){ + assertEquals("Number of doc level rules for test_windows category not correct", 5, noOfSigmaRuleMatches); + } + } else { + List> buckets = ((List>)(((Map)((Map)((Map)((List)((Map) executeResults.get("input_results")).get("results")).get(0)).get("aggregations")).get("result_agg")).get("buckets"))); + Integer docCount = buckets.stream().mapToInt(it -> (Integer)it.get("doc_count")).sum(); + assertEquals("Number of documents in buckets not correct", 5, docCount.intValue()); + List triggerResultBucketKeys = ((Map)((Map) ((Map)executeResults.get("trigger_results")).get(maxRuleId)).get("agg_result_buckets")).keySet().stream().collect(Collectors.toList()); + assertEquals("Number of triggers not correct", List.of("2", "3"), triggerResultBucketKeys); + } + } + // Call GetAlerts API + Map params = new HashMap<>(); + params.put("detector_id", detectorId); + + boolean totalAlertsEqualToExpected = waitUntil(() -> { + try { + Response getAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.ALERTS_BASE_URI, params, null); + Map getAlertsBody = asMap(getAlertsResponse); + // TODO enable asserts here when able + // one bucket level one custom doc level + int totalAlerts = (int) getAlertsBody.get("total_alerts"); + return totalAlerts == 12; + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + + assertTrue("Number of total alerts not correct", totalAlertsEqualToExpected); + } + public void testGetAlerts_byDetectorType_multipleDetectors_success() throws IOException, InterruptedException { String index1 = createTestIndex(randomIndex(), windowsIndexMapping()); @@ -471,10 +716,10 @@ public void testGetAlerts_byDetectorType_multipleDetectors_success() throws IOEx Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); // both req params and req body are supported createMappingRequest.setJsonEntity( - "{ \"index_name\":\"" + index1 + "\"," + - " \"rule_topic\":\"" + randomDetectorType() + "\", " + - " \"partial\":true" + - "}" + "{ \"index_name\":\"" + index1 + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" ); // index 2 String index2 = createTestIndex("netflow_test", netFlowMappings()); @@ -483,10 +728,10 @@ public void testGetAlerts_byDetectorType_multipleDetectors_success() throws IOEx createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); // both req params and req body are supported createMappingRequest.setJsonEntity( - "{ \"index_name\":\"" + index2 + "\"," + - " \"rule_topic\":\"netflow\", " + - " \"partial\":true" + - "}" + "{ \"index_name\":\"" + index2 + "\"," + + " \"rule_topic\":\"netflow\", " + + " \"partial\":true" + + "}" ); Response response = client().performRequest(createMappingRequest); @@ -501,23 +746,22 @@ public void testGetAlerts_byDetectorType_multipleDetectors_success() throws IOEx String createdId = responseBody.get("_id").toString(); String request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + createdId + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; List hits = executeSearch(Detector.DETECTORS_INDEX, request); SearchHit hit = hits.get(0); String monitorId1 = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); // Detector 2 - NETWORK DetectorInput inputNetflow = new DetectorInput("windows detector for security analytics", List.of("netflow_test"), Collections.emptyList(), - getPrePackagedRules("network").stream().map(DetectorRule::new).collect(Collectors.toList())); + getPrePackagedRules("network").stream().map(DetectorRule::new).collect(Collectors.toList()), List.of(Detector.DetectorType.NETWORK)); Detector detector2 = randomDetectorWithTriggers( - getPrePackagedRules("network"), - List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("network"), List.of(), List.of(), List.of(), List.of())), - Detector.DetectorType.NETWORK, - inputNetflow + getPrePackagedRules("network"), + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("network"), List.of(), List.of(), List.of(), List.of())), + inputNetflow ); createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector2)); @@ -528,12 +772,12 @@ public void testGetAlerts_byDetectorType_multipleDetectors_success() throws IOEx createdId = responseBody.get("_id").toString(); request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + createdId + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; hits = executeSearch(Detector.DETECTORS_INDEX, request); hit = hits.get(0); String monitorId2 = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); @@ -555,11 +799,11 @@ public void testGetAlerts_byDetectorType_multipleDetectors_success() throws IOEx client().performRequest(new Request("POST", "_refresh")); request = "{\n" + - " \"query\" : {\n" + - " \"match_all\":{\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; hits = new ArrayList<>(); while (hits.size() == 0) { @@ -598,10 +842,10 @@ public void testAlertHistoryRollover_maxAge() throws IOException, InterruptedExc Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); // both req params and req body are supported createMappingRequest.setJsonEntity( - "{ \"index_name\":\"" + index + "\"," + - " \"rule_topic\":\"" + randomDetectorType() + "\", " + - " \"partial\":true" + - "}" + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" ); Response response = client().performRequest(createMappingRequest); @@ -617,12 +861,12 @@ public void testAlertHistoryRollover_maxAge() throws IOException, InterruptedExc String createdId = responseBody.get("_id").toString(); String request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + createdId + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; List hits = executeSearch(Detector.DETECTORS_INDEX, request); SearchHit hit = hits.get(0); @@ -636,20 +880,20 @@ public void testAlertHistoryRollover_maxAge() throws IOException, InterruptedExc Assert.assertEquals(5, noOfSigmaRuleMatches); request = "{\n" + - " \"query\" : {\n" + - " \"match_all\":{\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; hits = new ArrayList<>(); while (hits.size() == 0) { hits = executeSearch(DetectorMonitorConfig.getAlertsIndex(randomDetectorType()), request); } - List alertIndices = getAlertIndices(detector.getDetectorType()); + List alertIndices = getAlertIndices(detector.getDetectorTypes().get(0)); while(alertIndices.size() < 3) { - alertIndices = getAlertIndices(detector.getDetectorType()); + alertIndices = getAlertIndices(detector.getDetectorTypes().get(0)); Thread.sleep(1000); } assertTrue("Did not find 3 alert indices", alertIndices.size() >= 3); @@ -667,10 +911,10 @@ public void testAlertHistoryRollover_maxAge_low_retention() throws IOException, Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); // both req params and req body are supported createMappingRequest.setJsonEntity( - "{ \"index_name\":\"" + index + "\"," + - " \"rule_topic\":\"" + randomDetectorType() + "\", " + - " \"partial\":true" + - "}" + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" ); Response response = client().performRequest(createMappingRequest); @@ -686,12 +930,12 @@ public void testAlertHistoryRollover_maxAge_low_retention() throws IOException, String createdId = responseBody.get("_id").toString(); String request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + createdId + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; List hits = executeSearch(Detector.DETECTORS_INDEX, request); SearchHit hit = hits.get(0); @@ -705,20 +949,20 @@ public void testAlertHistoryRollover_maxAge_low_retention() throws IOException, Assert.assertEquals(5, noOfSigmaRuleMatches); request = "{\n" + - " \"query\" : {\n" + - " \"match_all\":{\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; hits = new ArrayList<>(); while (hits.size() == 0) { hits = executeSearch(DetectorMonitorConfig.getAlertsIndex(randomDetectorType()), request); } - List alertIndices = getAlertIndices(detector.getDetectorType()); + List alertIndices = getAlertIndices(detector.getDetectorTypes().get(0)); while(alertIndices.size() < 3) { - alertIndices = getAlertIndices(detector.getDetectorType()); + alertIndices = getAlertIndices(detector.getDetectorTypes().get(0)); Thread.sleep(1000); } assertTrue("Did not find 3 alert indices", alertIndices.size() >= 3); @@ -727,7 +971,7 @@ public void testAlertHistoryRollover_maxAge_low_retention() throws IOException, updateClusterSetting(ALERT_HISTORY_RETENTION_PERIOD.getKey(), "1s"); while(alertIndices.size() != 1) { - alertIndices = getAlertIndices(detector.getDetectorType()); + alertIndices = getAlertIndices(detector.getDetectorTypes().get(0)); Thread.sleep(1000); } @@ -744,10 +988,10 @@ public void testAlertHistoryRollover_maxDocs() throws IOException, InterruptedEx Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); // both req params and req body are supported createMappingRequest.setJsonEntity( - "{ \"index_name\":\"" + index + "\"," + - " \"rule_topic\":\"" + randomDetectorType() + "\", " + - " \"partial\":true" + - "}" + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" ); Response response = client().performRequest(createMappingRequest); @@ -763,12 +1007,12 @@ public void testAlertHistoryRollover_maxDocs() throws IOException, InterruptedEx String detectorId = responseBody.get("_id").toString(); String request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; List hits = executeSearch(Detector.DETECTORS_INDEX, request); SearchHit hit = hits.get(0); @@ -784,11 +1028,11 @@ public void testAlertHistoryRollover_maxDocs() throws IOException, InterruptedEx Assert.assertEquals(5, noOfSigmaRuleMatches); request = "{\n" + - " \"query\" : {\n" + - " \"match_all\":{\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; hits = new ArrayList<>(); while (hits.size() == 0) { @@ -807,9 +1051,9 @@ public void testAlertHistoryRollover_maxDocs() throws IOException, InterruptedEx // Ack alert to move it to history index acknowledgeAlert(alertId, detectorId); - List alertIndices = getAlertIndices(detector.getDetectorType()); + List alertIndices = getAlertIndices(detector.getDetectorTypes().get(0)); while(alertIndices.size() < 3) { - alertIndices = getAlertIndices(detector.getDetectorType()); + alertIndices = getAlertIndices(detector.getDetectorTypes().get(0)); Thread.sleep(1000); } assertTrue("Did not find 3 alert indices", alertIndices.size() >= 3); @@ -826,10 +1070,10 @@ public void testGetAlertsFromAllIndices() throws IOException, InterruptedExcepti Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); // both req params and req body are supported createMappingRequest.setJsonEntity( - "{ \"index_name\":\"" + index + "\"," + - " \"rule_topic\":\"" + randomDetectorType() + "\", " + - " \"partial\":true" + - "}" + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" ); Response response = client().performRequest(createMappingRequest); @@ -845,12 +1089,12 @@ public void testGetAlertsFromAllIndices() throws IOException, InterruptedExcepti String detectorId = responseBody.get("_id").toString(); String request = "{\n" + - " \"query\" : {\n" + - " \"match\":{\n" + - " \"_id\": \"" + detectorId + "\"\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; List hits = executeSearch(Detector.DETECTORS_INDEX, request); SearchHit hit = hits.get(0); @@ -864,11 +1108,11 @@ public void testGetAlertsFromAllIndices() throws IOException, InterruptedExcepti Assert.assertEquals(5, noOfSigmaRuleMatches); request = "{\n" + - " \"query\" : {\n" + - " \"match_all\":{\n" + - " }\n" + - " }\n" + - "}"; + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; hits = new ArrayList<>(); while (hits.size() == 0) { @@ -887,10 +1131,10 @@ public void testGetAlertsFromAllIndices() throws IOException, InterruptedExcepti // Ack alert to move it to history index acknowledgeAlert(alertId, detectorId); - List alertIndices = getAlertIndices(detector.getDetectorType()); + List alertIndices = getAlertIndices(detector.getDetectorTypes().get(0)); // alertIndex + 2 alertHistory indices while(alertIndices.size() < 3) { - alertIndices = getAlertIndices(detector.getDetectorType()); + alertIndices = getAlertIndices(detector.getDetectorTypes().get(0)); Thread.sleep(1000); } assertTrue("Did not find 3 alert indices", alertIndices.size() >= 3); @@ -910,4 +1154,157 @@ public void testGetAlertsFromAllIndices() throws IOException, InterruptedExcepti // 1 from alertIndex and 1 from history index Assert.assertEquals(2, getAlertsBody.get("total_alerts")); } + + public void testGetAlertsFromAllIndicesMultipleDetectorTypes() throws IOException, InterruptedException { + String testOpCode = "Test"; + updateClusterSetting(ALERT_HISTORY_ROLLOVER_PERIOD.getKey(), "1s"); + updateClusterSetting(ALERT_HISTORY_MAX_DOCS.getKey(), "1"); + + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + List prepackagedRules = getRandomPrePackagedRules(); + String maxRuleId = createRule(randomAggregationRule("max", " > 3", testOpCode)); + String randomDocRuleId = createRule(randomRule(), "windows"); + + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(maxRuleId), new DetectorRule(randomDocRuleId)), + prepackagedRules.stream().map(DetectorRule::new).collect(Collectors.toList()), List.of(DetectorType.TEST_WINDOWS, DetectorType.WINDOWS)); + + Detector detector = randomDetectorWithTriggers(input, List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(windowsDetectorType(), randomDetectorType()), List.of(), List.of(), List.of(), List.of()))); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String detectorId = responseBody.get("_id").toString(); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + Map detectorMap = (HashMap)(hit.getSourceAsMap().get("detector")); + List monitorIds = ((List) (detectorMap).get("monitor_id")); + assertEquals(3, monitorIds.size()); + + indexDoc(index, "1", randomDoc(5, 3, testOpCode)); + indexDoc(index, "2", randomDoc(2, 3, testOpCode)); + indexDoc(index, "3", randomDoc(4, 3, testOpCode)); + indexDoc(index, "4", randomDoc(6, 2, testOpCode)); + + client().performRequest(new Request("POST", "_refresh")); + Map numberOfMonitorTypes = new HashMap<>(); + + Map docLevelMonitorIdPerCategory = ((Map)((Map)hit.getSourceAsMap().get("detector")).get("doc_monitor_id_per_category")); + // Swapping keys and values + Map ruleIdRuleCategoryMap = docLevelMonitorIdPerCategory.entrySet().stream().collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)); + + for(String monitorId: monitorIds) { + Map monitor = (Map)(entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + monitorId)))).get("monitor"); + numberOfMonitorTypes.merge(monitor.get("monitor_type"), 1, Integer::sum); + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + if (MonitorType.DOC_LEVEL_MONITOR.getValue().equals(monitor.get("monitor_type"))) { + String ruleCategory = ruleIdRuleCategoryMap.get(monitorId); + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + if (ruleCategory.toLowerCase(Locale.ROOT).equals(DetectorType.WINDOWS.getDetectorType().toLowerCase(Locale.ROOT))) { + assertEquals("Number of doc level rules for windows category not correct", 1, noOfSigmaRuleMatches); + } else if (ruleCategory.toLowerCase(Locale.ROOT).equals(DetectorType.TEST_WINDOWS.getDetectorType().toLowerCase(Locale.ROOT))){ + assertEquals("Number of doc level rules for test_windows category not correct", 5, noOfSigmaRuleMatches); + } + } else { + List> buckets = ((List>)(((Map)((Map)((Map)((List)((Map) executeResults.get("input_results")).get("results")).get(0)).get("aggregations")).get("result_agg")).get("buckets"))); + Integer docCount = buckets.stream().mapToInt(it -> (Integer)it.get("doc_count")).sum(); + assertEquals("Number of documents in buckets not correct", 4, docCount.intValue()); + List triggerResultBucketKeys = ((Map)((Map) ((Map)executeResults.get("trigger_results")).get(maxRuleId)).get("agg_result_buckets")).keySet().stream().collect(Collectors.toList()); + assertEquals("Trigger results not correct", List.of("2", "3"), triggerResultBucketKeys); + } + } + + request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + hits = new ArrayList<>(); + + while (hits.size() == 0) { + hits = executeSearch(DetectorMonitorConfig.getAlertsIndex("windows"), request); + } + // Call GetAlerts API + Map params = new HashMap<>(); + params.put("detectorType", "windows"); + Response getAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.ALERTS_BASE_URI, params, null); + Map getAlertsBody = asMap(getAlertsResponse); + // TODO enable asserts here when able + assertEquals("Number of total alerts for windows category not correct", 4, getAlertsBody.get("total_alerts")); + + List> alerts = (ArrayList>) getAlertsBody.get("alerts"); + + for(Map alert: alerts) { + String alertId =(String) alert.get("id"); + // Ack alert to move it to history index + acknowledgeAlert(alertId, detectorId); + } + + hits = new ArrayList<>(); + while (hits.size() == 0) { + hits = executeSearch(DetectorMonitorConfig.getAlertsIndex("test_windows"), request); + } + // Call GetAlerts API + params.put("detectorType", "test_windows"); + getAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.ALERTS_BASE_URI, params, null); + getAlertsBody = asMap(getAlertsResponse); + // TODO enable asserts here when able + Assert.assertEquals("Number of total alerts for test_windows category not correct", 6, getAlertsBody.get("total_alerts")); + /** TODO - uncomment once https://github.com/opensearch-project/alerting/pull/753 is merged + alerts = (ArrayList>) getAlertsBody.get("alerts"); + + for(Map alert: alerts) { + String alertId =(String) alert.get("id"); + // Ack alert to move it to history index + acknowledgeAlert(alertId, detectorId); + } + + List alertIndices = getAlertIndices(detector.getDetectorTypes().get(0)); + // alertIndex + 2 alertHistory indices + while(alertIndices.size() < 3) { + alertIndices = getAlertIndices(detector.getDetectorTypes().get(0)); + Thread.sleep(1000); + } + assertTrue("Did not find 3 alert indices", alertIndices.size() >= 3); + + // Index another doc to generate new alert in alertIndex + indexDoc(index, "5", randomDoc(1, 1, testOpCode)); + + for(String monitorId: monitorIds) { + executeAlertingMonitor(monitorId, Collections.emptyMap()); + } + + client().performRequest(new Request("POST", DetectorMonitorConfig.getAlertsIndex(randomDetectorType()) + "/_refresh")); + getAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.ALERTS_BASE_URI, params, null); + getAlertsBody = asMap(getAlertsResponse); + + assertEquals("Number of alerts not correct", 9, getAlertsBody.get("total_alerts"));**/ + } } \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/SecureAlertsRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/alerts/SecureAlertsRestApiIT.java index ff6cba8bb..09b88b7f4 100644 --- a/src/test/java/org/opensearch/securityanalytics/alerts/SecureAlertsRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/alerts/SecureAlertsRestApiIT.java @@ -96,7 +96,7 @@ public void testGetAlerts_byDetectorId_success() throws IOException { Action triggerAction = randomAction(createDestination()); Detector detector = randomDetectorWithInputsAndTriggers(List.of(new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(createdId)), - getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList()))), + getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList()), Collections.emptyList())), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(), List.of(createdId), List.of(), List.of("attack.defense_evasion"), List.of(triggerAction)))); createResponse = makeRequest(userClient, "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); @@ -208,7 +208,6 @@ public void testGetAlerts_byDetectorId_success() throws IOException { } finally { tryDeletingRole(TEST_HR_ROLE); } - } diff --git a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java index 7b34b7d94..4224b63d3 100644 --- a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java +++ b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java @@ -6,30 +6,38 @@ package org.opensearch.securityanalytics.findings; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import org.apache.http.HttpStatus; import org.junit.Assert; import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; +import org.opensearch.commons.alerting.model.Monitor.MonitorType; import org.opensearch.rest.RestStatus; import org.opensearch.search.SearchHit; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; import org.opensearch.securityanalytics.model.Detector; +import org.opensearch.securityanalytics.model.Detector.DetectorType; import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.model.DetectorRule; import org.opensearch.securityanalytics.model.DetectorTrigger; import static org.opensearch.securityanalytics.TestHelpers.netFlowMappings; +import static org.opensearch.securityanalytics.TestHelpers.randomAggregationRule; import static org.opensearch.securityanalytics.TestHelpers.randomDetectorType; +import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithInputs; import static org.opensearch.securityanalytics.TestHelpers.randomDetectorWithTriggers; import static org.opensearch.securityanalytics.TestHelpers.randomDoc; import static org.opensearch.securityanalytics.TestHelpers.randomIndex; +import static org.opensearch.securityanalytics.TestHelpers.randomRule; import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_INDEX_MAX_AGE; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_MAX_DOCS; @@ -147,12 +155,360 @@ public void testGetFindings_byDetectorType_oneDetector_success() throws IOExcept Assert.assertEquals(5, noOfSigmaRuleMatches); // Call GetFindings API Map params = new HashMap<>(); - params.put("detectorType", detector.getDetectorType()); + params.put("detectorType", detector.getDetectorTypes().get(0)); Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); Map getFindingsBody = entityAsMap(getFindingsResponse); Assert.assertEquals(1, getFindingsBody.get("total_findings")); } + public void testGetFindings_byDetectorType_oneDetector_multipleDetectorTypes_success() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + "windows" + "\", " + + " \"partial\":true" + + "}" + ); + + response = client().performRequest(createMappingRequest); + + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + // Fetching 10 rules -> 5 from test_windows and 5 from windows category + List prepackagedRules = getRandomPrePackagedRules(); + String randomDocRuleId = createRule(randomRule(), "windows"); + + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(randomDocRuleId)), + prepackagedRules.stream().map(DetectorRule::new).collect(Collectors.toList()), List.of(DetectorType.TEST_WINDOWS, DetectorType.WINDOWS)); + + Detector detector = randomDetectorWithInputs(List.of(input)); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + String createdId = responseBody.get("_id").toString(); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + Map detectorMap = (HashMap)(hit.getSourceAsMap().get("detector")); + List inputArr = detectorMap.get("inputs"); + + assertEquals("Number of custom rules not correct", 1, ((Map>) inputArr.get(0)).get("detector_input").get("custom_rules").size()); + + List monitorIds = ((List) (detectorMap).get("monitor_id")); + + assertEquals("Number of monitors not correct", 2, monitorIds.size()); + + Map docLevelMonitorIdPerCategory = ((Map)((Map)hit.getSourceAsMap().get("detector")).get("doc_monitor_id_per_category")); + // Swapping keys and values + Map ruleIdRuleCategoryMap = docLevelMonitorIdPerCategory.entrySet().stream().collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)); + + String infoOpCode = "Info"; + String testOpCode = "Test"; + indexDoc(index, "1", randomDoc(2, 4, infoOpCode)); + indexDoc(index, "2", randomDoc(3, 4, infoOpCode)); + indexDoc(index, "3", randomDoc(1, 4, infoOpCode)); + indexDoc(index, "4", randomDoc(5, 3, testOpCode)); + indexDoc(index, "5", randomDoc(2, 3, testOpCode)); + indexDoc(index, "6", randomDoc(4, 3, testOpCode)); + indexDoc(index, "7", randomDoc(6, 2, testOpCode)); + indexDoc(index, "8", randomDoc(1, 1, testOpCode)); + + Set expectedDocIds = Set.of("1", "2", "3", "4", "5", "6", "7", "8"); + + for(String monitorId: monitorIds) { + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + String ruleCategory = ruleIdRuleCategoryMap.get(monitorId); + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + + if (ruleCategory.toLowerCase(Locale.ROOT).equals(DetectorType.WINDOWS.getDetectorType().toLowerCase(Locale.ROOT))) { + Assert.assertEquals(1, noOfSigmaRuleMatches); + // Call GetFindings API + Map params = new HashMap<>(); + params.put("detectorType", DetectorType.WINDOWS.getDetectorType()); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + + assertEquals("Number of total findings for windows category not correct", 8, getFindingsBody.get("total_findings")); + assertFindingsPerExecutedDocLevelMonitor(getFindingsBody, expectedDocIds); + + } else { + assertEquals("Number of doc level rules for test_windows category not correct", 5, noOfSigmaRuleMatches); + // Call GetFindings API + Map params = new HashMap<>(); + params.put("detectorType", DetectorType.TEST_WINDOWS.getDetectorType()); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + + assertEquals("Number of total findings for test_windows category not correct", 8, getFindingsBody.get("total_findings")); + assertFindingsPerExecutedDocLevelMonitor(getFindingsBody, expectedDocIds); + } + } + } + public void testGetFindings_byDetectorType_multipleDetectorTypes_FindingForOneLogType_success() throws IOException { + String infoOpCode = "Info"; + String testOpCode = "Test"; + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + "windows" + "\", " + + " \"partial\":true" + + "}" + ); + + response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + // Fetching 6 rules -> 5 from test_windows and 1 from windows category (custom doc rule) + List prepackagedRules = getRandomPrePackagedRules(); + String maxRuleId = createRule(randomAggregationRule("max", " > 3", testOpCode)); + + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(maxRuleId)), + prepackagedRules.stream().map(DetectorRule::new).collect(Collectors.toList()), List.of(DetectorType.TEST_WINDOWS, DetectorType.WINDOWS)); + + Detector detector = randomDetectorWithInputs(List.of(input)); + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String detectorId = responseBody.get("_id").toString(); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + Map detectorMap = (HashMap)(hit.getSourceAsMap().get("detector")); + List inputArr = detectorMap.get("inputs"); + assertEquals("Number of custom rules not correct", 1, ((Map>) inputArr.get(0)).get("detector_input").get("custom_rules").size()); + List monitorIds = ((List) (detectorMap).get("monitor_id")); + // 2 doc level monitors - one per each category: test_windows and windows + // 1 bucket level monitor - for windows category + assertEquals("Number of monitors not correct", 2, monitorIds.size()); + + Map docLevelMonitorIdPerCategory = ((Map)((Map)hit.getSourceAsMap().get("detector")).get("doc_monitor_id_per_category")); + // Swapping keys and values + Map ruleIdRuleCategoryMap = docLevelMonitorIdPerCategory.entrySet().stream().collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)); + + + indexDoc(index, "1", randomDoc(2, 4, infoOpCode)); + indexDoc(index, "2", randomDoc(3, 4, infoOpCode)); + indexDoc(index, "3", randomDoc(1, 4, infoOpCode)); + indexDoc(index, "4", randomDoc(2, 3, testOpCode)); + indexDoc(index, "5", randomDoc(2, 3, testOpCode)); + indexDoc(index, "6", randomDoc(1, 3, testOpCode)); + indexDoc(index, "7", randomDoc(1, 2, testOpCode)); + indexDoc(index, "8", randomDoc(1, 1, testOpCode)); + Set expectedDocIds = Set.of("1", "2", "3", "4", "5", "6", "7", "8"); + + Map numberOfMonitorTypes = new HashMap<>(); + + for(String monitorId: monitorIds) { + Map monitor = (Map)(entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + monitorId)))).get("monitor"); + numberOfMonitorTypes.merge(monitor.get("monitor_type"), 1, Integer::sum); + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + if (MonitorType.DOC_LEVEL_MONITOR.getValue().equals(monitor.get("monitor_type"))) { + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + assertEquals("Number of doc level rules not correct", 5, noOfSigmaRuleMatches); + + // Call GetFindings API + Map params = new HashMap<>(); + params.put("detectorType", DetectorType.TEST_WINDOWS.getDetectorType()); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + + assertEquals("Number of total findings for test_windows category not correct", 8, getFindingsBody.get("total_findings")); + assertFindingsPerExecutedDocLevelMonitor(getFindingsBody, expectedDocIds); + } else { + List> buckets = ((List>)(((Map)((Map)((Map)((List)((Map) executeResults.get("input_results")).get("results")).get(0)).get("aggregations")).get("result_agg")).get("buckets"))); + Integer docCount = buckets.stream().mapToInt(it -> (Integer)it.get("doc_count")).sum(); + assertEquals("Number of bucket level monitors not correct", 5, docCount.intValue()); + List triggerResultBucketKeys = ((Map)((Map) ((Map)executeResults.get("trigger_results")).get(maxRuleId)).get("agg_result_buckets")).keySet().stream().collect(Collectors.toList()); + assertEquals(Collections.emptyList(), triggerResultBucketKeys); + + // Call GetFindings API + Map params = new HashMap<>(); + params.put("detectorType", DetectorType.WINDOWS.getDetectorType()); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + assertEquals("Number of total findings for windows category not correct", 0, getFindingsBody.get("total_findings")); + } + } + assertEquals("Number of bucket level monitors not correct", 1, numberOfMonitorTypes.get(MonitorType.BUCKET_LEVEL_MONITOR.getValue()).intValue()); + assertEquals("Number of doc level monitors not correct", 1, numberOfMonitorTypes.get(MonitorType.DOC_LEVEL_MONITOR.getValue()).intValue()); + } + + public void testGetFindings_byDetectorId_oneDetector_multipleDetectorTypes_success() throws IOException { + String infoOpCode = "Info"; + String testOpCode = "Test"; + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + "windows" + "\", " + + " \"partial\":true" + + "}" + ); + + response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + // Fetching 6 rules -> 5 from test_windows and 1 from windows category (custom doc rule) + List prepackagedRules = getRandomPrePackagedRules(); + String randomDocRuleId = createRule(randomRule(), "windows"); + String maxRuleId = createRule(randomAggregationRule("max", " > 3", testOpCode)); + + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(randomDocRuleId), new DetectorRule(maxRuleId)), + prepackagedRules.stream().map(DetectorRule::new).collect(Collectors.toList()), List.of(DetectorType.TEST_WINDOWS, DetectorType.WINDOWS)); + + Detector detector = randomDetectorWithInputs(List.of(input)); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String detectorId = responseBody.get("_id").toString(); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + Map detectorMap = (HashMap)(hit.getSourceAsMap().get("detector")); + List inputArr = detectorMap.get("inputs"); + assertEquals("Number of custom rules not correct", 2, ((Map>) inputArr.get(0)).get("detector_input").get("custom_rules").size()); + List monitorIds = ((List) (detectorMap).get("monitor_id")); + // 2 doc level monitors - one per each category: test_windows and windows + // 1 bucket level monitor - for windows category + assertEquals("Number of monitors not correct", 3, monitorIds.size()); + + Map docLevelMonitorIdPerCategory = ((Map)((Map)hit.getSourceAsMap().get("detector")).get("doc_monitor_id_per_category")); + // Swapping keys and values + Map ruleIdRuleCategoryMap = docLevelMonitorIdPerCategory.entrySet().stream().collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)); + + + indexDoc(index, "1", randomDoc(2, 4, infoOpCode)); + indexDoc(index, "2", randomDoc(3, 4, infoOpCode)); + indexDoc(index, "3", randomDoc(1, 4, infoOpCode)); + indexDoc(index, "4", randomDoc(5, 3, testOpCode)); + indexDoc(index, "5", randomDoc(2, 3, testOpCode)); + indexDoc(index, "6", randomDoc(4, 3, testOpCode)); + indexDoc(index, "7", randomDoc(6, 2, testOpCode)); + indexDoc(index, "8", randomDoc(1, 1, testOpCode)); + + Map numberOfMonitorTypes = new HashMap<>(); + + for(String monitorId: monitorIds) { + Map monitor = (Map)(entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + monitorId)))).get("monitor"); + numberOfMonitorTypes.merge(monitor.get("monitor_type"), 1, Integer::sum); + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + if (MonitorType.DOC_LEVEL_MONITOR.getValue().equals(monitor.get("monitor_type"))) { + String ruleCategory = ruleIdRuleCategoryMap.get(monitorId); + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + if (ruleCategory.toLowerCase(Locale.ROOT).equals(DetectorType.WINDOWS.getDetectorType().toLowerCase(Locale.ROOT))) { + assertEquals("Number of doc level rules for windows category not correct", 1, noOfSigmaRuleMatches); + } else if(ruleCategory.toLowerCase(Locale.ROOT).equals(DetectorType.TEST_WINDOWS.getDetectorType().toLowerCase(Locale.ROOT))){ + assertEquals("Number of doc level rules for test_windows category not correct", 5, noOfSigmaRuleMatches); + } + } else { + List> buckets = ((List>)(((Map)((Map)((Map)((List)((Map) executeResults.get("input_results")).get("results")).get(0)).get("aggregations")).get("result_agg")).get("buckets"))); + Integer docCount = buckets.stream().mapToInt(it -> (Integer)it.get("doc_count")).sum(); + assertEquals("Number of documents in buckets not correct", 5, docCount.intValue()); + List triggerResultBucketKeys = ((Map)((Map) ((Map)executeResults.get("trigger_results")).get(maxRuleId)).get("agg_result_buckets")).keySet().stream().collect(Collectors.toList()); + assertEquals("Trigger result not correct", List.of("2", "3"), triggerResultBucketKeys); + } + } + + assertEquals("Number of bucket level monitors not correct", 1, numberOfMonitorTypes.get(MonitorType.BUCKET_LEVEL_MONITOR.getValue()).intValue()); + assertEquals("Number of doc level monitors not correct", 2, numberOfMonitorTypes.get(MonitorType.DOC_LEVEL_MONITOR.getValue()).intValue()); + + Map params = new HashMap<>(); + params.put("detector_id", detectorId); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + // 8 findings from prepackaged doc rules + // 8 findings from custom created doc level rule + // 1 finding from custom aggregation rule + assertEquals("Number of total findings not correct", 17, getFindingsBody.get("total_findings")); + } + public void testGetFindings_byDetectorType_success() throws IOException { String index1 = createTestIndex(randomIndex(), windowsIndexMapping()); @@ -205,11 +561,10 @@ public void testGetFindings_byDetectorType_success() throws IOException { String monitorId1 = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); // Detector 2 - NETWORK DetectorInput inputNetflow = new DetectorInput("windows detector for security analytics", List.of("netflow_test"), Collections.emptyList(), - getPrePackagedRules("network").stream().map(DetectorRule::new).collect(Collectors.toList())); + getPrePackagedRules("network").stream().map(DetectorRule::new).collect(Collectors.toList()), List.of(Detector.DetectorType.NETWORK)); Detector detector2 = randomDetectorWithTriggers( getPrePackagedRules("network"), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("network"), List.of(), List.of(), List.of(), List.of())), - Detector.DetectorType.NETWORK, inputNetflow ); @@ -251,18 +606,19 @@ public void testGetFindings_byDetectorType_success() throws IOException { // Call GetFindings API for first detector Map params = new HashMap<>(); - params.put("detectorType", detector1.getDetectorType()); + params.put("detectorType", detector1.getDetectorTypes().get(0)); Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); Map getFindingsBody = entityAsMap(getFindingsResponse); Assert.assertEquals(1, getFindingsBody.get("total_findings")); // Call GetFindings API for second detector params.clear(); - params.put("detectorType", detector2.getDetectorType()); + params.put("detectorType", detector2.getDetectorTypes().get(0)); getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); getFindingsBody = entityAsMap(getFindingsResponse); Assert.assertEquals(1, getFindingsBody.get("total_findings")); } + public void testGetFindings_rolloverByMaxAge_success() throws IOException, InterruptedException { updateClusterSetting(FINDING_HISTORY_ROLLOVER_PERIOD.getKey(), "1s"); @@ -318,9 +674,9 @@ public void testGetFindings_rolloverByMaxAge_success() throws IOException, Inter Map getFindingsBody = entityAsMap(getFindingsResponse); Assert.assertEquals(1, getFindingsBody.get("total_findings")); - List findingIndices = getFindingIndices(detector.getDetectorType()); + List findingIndices = getFindingIndices(detector.getDetectorTypes().get(0)); while(findingIndices.size() < 2) { - findingIndices = getFindingIndices(detector.getDetectorType()); + findingIndices = getFindingIndices(detector.getDetectorTypes().get(0)); Thread.sleep(1000); } assertTrue("Did not find 3 alert indices", findingIndices.size() >= 2); @@ -381,9 +737,9 @@ public void testGetFindings_rolloverByMaxDoc_success() throws IOException, Inter Map getFindingsBody = entityAsMap(getFindingsResponse); Assert.assertEquals(1, getFindingsBody.get("total_findings")); - List findingIndices = getFindingIndices(detector.getDetectorType()); + List findingIndices = getFindingIndices(detector.getDetectorTypes().get(0)); while(findingIndices.size() < 2) { - findingIndices = getFindingIndices(detector.getDetectorType()); + findingIndices = getFindingIndices(detector.getDetectorTypes().get(0)); Thread.sleep(1000); } assertTrue("Did not find 3 alert indices", findingIndices.size() >= 2); @@ -444,9 +800,9 @@ public void testGetFindings_rolloverByMaxDoc_short_retention_success() throws IO Map getFindingsBody = entityAsMap(getFindingsResponse); Assert.assertEquals(1, getFindingsBody.get("total_findings")); - List findingIndices = getFindingIndices(detector.getDetectorType()); + List findingIndices = getFindingIndices(detector.getDetectorTypes().get(0)); while(findingIndices.size() < 2) { - findingIndices = getFindingIndices(detector.getDetectorType()); + findingIndices = getFindingIndices(detector.getDetectorTypes().get(0)); Thread.sleep(1000); } assertTrue("Did not find 3 findings indices", findingIndices.size() >= 2); @@ -454,7 +810,7 @@ public void testGetFindings_rolloverByMaxDoc_short_retention_success() throws IO updateClusterSetting(FINDING_HISTORY_RETENTION_PERIOD.getKey(), "1s"); updateClusterSetting(FINDING_HISTORY_MAX_DOCS.getKey(), "1000"); while(findingIndices.size() != 1) { - findingIndices = getFindingIndices(detector.getDetectorType()); + findingIndices = getFindingIndices(detector.getDetectorTypes().get(0)); Thread.sleep(1000); } @@ -473,4 +829,13 @@ public void testGetFindings_rolloverByMaxDoc_short_retention_success() throws IO getFindingsBody = entityAsMap(getFindingsResponse); Assert.assertEquals(1, getFindingsBody.get("total_findings")); } + + private static void assertFindingsPerExecutedDocLevelMonitor(Map getFindingsBody, Set expectedDocIds) { + List> findings = (List) getFindingsBody.get("findings"); + List relatedDocFinding = new ArrayList<>(); + for(Map finding : findings) { + relatedDocFinding.addAll((List) finding.get("related_doc_ids")); + } + assertTrue(expectedDocIds.containsAll(relatedDocFinding)); + } } diff --git a/src/test/java/org/opensearch/securityanalytics/findings/FindingServiceTests.java b/src/test/java/org/opensearch/securityanalytics/findings/FindingServiceTests.java index 6ad0b5a14..ec9131b84 100644 --- a/src/test/java/org/opensearch/securityanalytics/findings/FindingServiceTests.java +++ b/src/test/java/org/opensearch/securityanalytics/findings/FindingServiceTests.java @@ -28,6 +28,7 @@ import org.opensearch.securityanalytics.action.GetFindingsResponse; import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig; import org.opensearch.securityanalytics.model.Detector; +import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.test.OpenSearchTestCase; @@ -53,9 +54,8 @@ public void testGetFindings_success() { new CronSchedule("31 * * * *", ZoneId.of("Asia/Kolkata"), Instant.ofEpochSecond(1538164858L)), Instant.now(), Instant.now(), - Detector.DetectorType.OTHERS_APPLICATION, null, - List.of(), + List.of(new DetectorInput("", Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), List.of(Detector.DetectorType.OTHERS_APPLICATION))), List.of(), List.of("monitor_id1", "monitor_id2"), DetectorMonitorConfig.getRuleIndex(Detector.DetectorType.OTHERS_APPLICATION.getDetectorType()), @@ -64,6 +64,7 @@ public void testGetFindings_success() { null, null, DetectorMonitorConfig.getFindingsIndex(Detector.DetectorType.OTHERS_APPLICATION.getDetectorType()), + Collections.emptyMap(), Collections.emptyMap() ); GetDetectorResponse getDetectorResponse = new GetDetectorResponse("detector_id123", 1L, RestStatus.OK, detector); @@ -169,9 +170,8 @@ public void testGetFindings_getFindingsByMonitorIdFailure() { new CronSchedule("31 * * * *", ZoneId.of("Asia/Kolkata"), Instant.ofEpochSecond(1538164858L)), Instant.now(), Instant.now(), - Detector.DetectorType.OTHERS_APPLICATION, null, - List.of(), + List.of(new DetectorInput("", Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), List.of(Detector.DetectorType.OTHERS_APPLICATION))), List.of(), List.of("monitor_id1", "monitor_id2"), DetectorMonitorConfig.getRuleIndex(Detector.DetectorType.OTHERS_APPLICATION.getDetectorType()), @@ -180,6 +180,7 @@ public void testGetFindings_getFindingsByMonitorIdFailure() { null, null, DetectorMonitorConfig.getFindingsIndex(Detector.DetectorType.OTHERS_APPLICATION.getDetectorType()), + Collections.emptyMap(), Collections.emptyMap() ); GetDetectorResponse getDetectorResponse = new GetDetectorResponse("detector_id123", 1L, RestStatus.OK, detector); diff --git a/src/test/java/org/opensearch/securityanalytics/findings/SecureFindingRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/findings/SecureFindingRestApiIT.java index 17add4956..0463a34a4 100644 --- a/src/test/java/org/opensearch/securityanalytics/findings/SecureFindingRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/findings/SecureFindingRestApiIT.java @@ -221,18 +221,16 @@ public void testGetFindings_byDetectorType_success() throws IOException { " }\n" + " }\n" + "}"; - List hits = executeSearch(Detector.DETECTORS_INDEX, request); - SearchHit hit = hits.get(0); - String monitorId1 = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); - // Detector 2 - NETWORK - DetectorInput inputNetflow = new DetectorInput("windows detector for security analytics", List.of("netflow_test"), Collections.emptyList(), - getPrePackagedRules("network").stream().map(DetectorRule::new).collect(Collectors.toList())); - Detector detector2 = randomDetectorWithTriggers( + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + String monitorId1 = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + // Detector 2 - NETWORK + DetectorInput inputNetflow = new DetectorInput("windows detector for security analytics", List.of("netflow_test"), Collections.emptyList(), + getPrePackagedRules("network").stream().map(DetectorRule::new).collect(Collectors.toList()), List.of(Detector.DetectorType.NETWORK)); + Detector detector2 = randomDetectorWithTriggers( getPrePackagedRules("network"), - List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("network"), List.of(), List.of(), List.of(), List.of())), - Detector.DetectorType.NETWORK, - inputNetflow - ); + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of("network"), List.of(), List.of(), List.of(), List.of())), inputNetflow + ); createResponse = makeRequest(userClient, "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector2)); Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); @@ -280,13 +278,13 @@ public void testGetFindings_byDetectorType_success() throws IOException { // Call GetFindings API for first detector Map params = new HashMap<>(); - params.put("detectorType", detector1.getDetectorType()); + params.put("detectorType", detector1.getDetectorTypes().get(0)); Response getFindingsResponse = makeRequest(userReadOnlyClient, "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); Map getFindingsBody = entityAsMap(getFindingsResponse); Assert.assertEquals(1, getFindingsBody.get("total_findings")); // Call GetFindings API for second detector params.clear(); - params.put("detectorType", detector2.getDetectorType()); + params.put("detectorType", detector2.getDetectorTypes().get(0)); getFindingsResponse = makeRequest(userReadOnlyClient, "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); getFindingsBody = entityAsMap(getFindingsResponse); Assert.assertEquals(1, getFindingsBody.get("total_findings")); diff --git a/src/test/java/org/opensearch/securityanalytics/model/WriteableTests.java b/src/test/java/org/opensearch/securityanalytics/model/WriteableTests.java index 2326b541d..daa34ea66 100644 --- a/src/test/java/org/opensearch/securityanalytics/model/WriteableTests.java +++ b/src/test/java/org/opensearch/securityanalytics/model/WriteableTests.java @@ -21,7 +21,7 @@ public class WriteableTests extends OpenSearchTestCase { public void testDetectorAsStream() throws IOException { Detector detector = randomDetector(List.of()); - detector.setInputs(List.of(new DetectorInput("", List.of(), List.of(), List.of()))); + detector.setInputs(List.of(new DetectorInput("", List.of(), List.of(), List.of(), List.of()))); BytesStreamOutput out = new BytesStreamOutput(); detector.writeTo(out); StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); diff --git a/src/test/java/org/opensearch/securityanalytics/model/XContentTests.java b/src/test/java/org/opensearch/securityanalytics/model/XContentTests.java index ce1ae3806..b6fa093b8 100644 --- a/src/test/java/org/opensearch/securityanalytics/model/XContentTests.java +++ b/src/test/java/org/opensearch/securityanalytics/model/XContentTests.java @@ -34,7 +34,6 @@ public void testDetectorParsing() throws IOException { public void testDetectorParsingWithNoName() { String detectorStringWithoutName = "{\n" + " \"type\": \"detector\",\n" + - " \"detector_type\": \"WINDOWS\",\n" + " \"user\": {\n" + " \"name\": \"JPXeGWmlMP\",\n" + " \"backend_roles\": [\n" + @@ -66,7 +65,8 @@ public void testDetectorParsingWithNoName() { " \"windows\"\n" + " ],\n" + " \"custom_rules\": [],\n" + - " \"pre_packaged_rules\": []\n" + + " \"pre_packaged_rules\": [],\n" + + " \"detector_types\": [\"WINDOWS\"]\n" + " }\n" + " }\n" + " ],\n" + @@ -106,7 +106,6 @@ public void testDetectorParsingWithNoSchedule() { String detectorStringWithoutSchedule = "{\n" + " \"type\": \"detector\",\n" + " \"name\": \"BCIocIalTX\",\n" + - " \"detector_type\": \"WINDOWS\",\n" + " \"user\": {\n" + " \"name\": \"JPXeGWmlMP\",\n" + " \"backend_roles\": [\n" + @@ -132,7 +131,8 @@ public void testDetectorParsingWithNoSchedule() { " \"windows\"\n" + " ],\n" + " \"custom_rules\": [],\n" + - " \"pre_packaged_rules\": []\n" + + " \"pre_packaged_rules\": [],\n" + + " \"detector_types\": [\"WINDOWS\"]\n" + " }\n" + " }\n" + " ],\n" + diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java index 10e909b1c..76cad66a2 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorMonitorRestApiIT.java @@ -11,11 +11,13 @@ import static org.opensearch.securityanalytics.TestHelpers.randomDoc; import static org.opensearch.securityanalytics.TestHelpers.randomIndex; import static org.opensearch.securityanalytics.TestHelpers.randomRule; +import static org.opensearch.securityanalytics.TestHelpers.windowsDetectorType; import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -35,6 +37,7 @@ import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig; import org.opensearch.securityanalytics.model.Detector; +import org.opensearch.securityanalytics.model.Detector.DetectorType; import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.model.DetectorRule; import org.opensearch.securityanalytics.model.Rule; @@ -78,7 +81,7 @@ public void testRemoveDocLevelRuleAddAggregationRules_verifyFindings_success() t "}"; SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); - assertEquals(5, response.getHits().getTotalHits().value); + assertEquals("Number of total hits not correct", 5, response.getHits().getTotalHits().value); Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); @@ -96,7 +99,7 @@ public void testRemoveDocLevelRuleAddAggregationRules_verifyFindings_success() t Map detectorAsMap = (Map) hit.getSourceAsMap().get("detector"); List monitorIds = (List) (detectorAsMap).get("monitor_id"); - assertEquals(1, monitorIds.size()); + assertEquals("Number of monitors not correct", 1, monitorIds.size()); String monitorId = monitorIds.get(0); String monitorType = ((Map) entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + monitorId))).get("monitor")).get("monitor_type"); @@ -104,11 +107,11 @@ public void testRemoveDocLevelRuleAddAggregationRules_verifyFindings_success() t assertEquals(MonitorType.DOC_LEVEL_MONITOR.getValue(), monitorType); // Create aggregation rules - String sumRuleId = createRule(randomAggregationRule( "sum", " > 2")); - String avgTermRuleId = createRule(randomAggregationRule( "avg", " > 1")); + String sumRuleId = createRule(randomAggregationRule( "sum", " > 2"), "test_windows"); + String avgTermRuleId = createRule(randomAggregationRule( "avg", " > 1"), "test_windows"); // Update detector and empty doc level rules so detector contains only one aggregation rule DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(sumRuleId), new DetectorRule(avgTermRuleId)), - Collections.emptyList()); + Collections.emptyList(), Collections.emptyList()); Detector updatedDetector = randomDetectorWithInputs(List.of(input)); Response updateResponse = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId, Collections.emptyMap(), toHttpEntity(updatedDetector)); @@ -131,7 +134,9 @@ public void testRemoveDocLevelRuleAddAggregationRules_verifyFindings_success() t // Execute two bucket level monitors for(String id: monitorIds){ monitorType = ((Map) entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + id))).get("monitor")).get("monitor_type"); - Assert.assertEquals(MonitorType.BUCKET_LEVEL_MONITOR.getValue(), monitorType); + + assertEquals("Invalid monitor type", MonitorType.BUCKET_LEVEL_MONITOR.getValue(), monitorType); + executeAlertingMonitor(id, Collections.emptyMap()); } // verify bucket level monitor findings @@ -143,11 +148,9 @@ public void testRemoveDocLevelRuleAddAggregationRules_verifyFindings_success() t assertNotNull(getFindingsBody); assertEquals(2, getFindingsBody.get("total_findings")); - List aggRuleIds = List.of(sumRuleId, avgTermRuleId); - - List> findings = (List)getFindingsBody.get("findings"); + List> findings = (List) getFindingsBody.get("findings"); for(Map finding : findings) { - Set aggRulesFinding = ((List>)finding.get("queries")).stream().map(it -> it.get("id").toString()).collect( + Set aggRulesFinding = ((List>) finding.get("queries")).stream().map(it -> it.get("id").toString()).collect( Collectors.toSet()); // Bucket monitor finding will have one rule String aggRuleId = aggRulesFinding.iterator().next(); @@ -155,12 +158,12 @@ public void testRemoveDocLevelRuleAddAggregationRules_verifyFindings_success() t assertTrue(aggRulesFinding.contains(aggRuleId)); List findingDocs = (List)finding.get("related_doc_ids"); - Assert.assertEquals(2, findingDocs.size()); + assertEquals("Number of found document not correct", 2, findingDocs.size()); assertTrue(Arrays.asList("1", "2").containsAll(findingDocs)); } String findingDetectorId = ((Map)((List)getFindingsBody.get("findings")).get(0)).get("detectorId").toString(); - assertEquals(detectorId, findingDetectorId); + assertEquals("Detector id is not as expected", detectorId, findingDetectorId); String findingIndex = ((Map)((List)getFindingsBody.get("findings")).get(0)).get("index").toString(); assertEquals(index, findingIndex); @@ -190,10 +193,10 @@ public void testReplaceAggregationRuleWithDocRule_verifyFindings_success() throw assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode()); - String maxRuleId = createRule(randomAggregationRule( "max", " > 2")); + String maxRuleId = createRule(randomAggregationRule( "max", " > 2"), "test_windows"); List detectorRules = List.of(new DetectorRule(maxRuleId)); DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), detectorRules, - Collections.emptyList()); + Collections.emptyList(), Collections.emptyList()); Detector detector = randomDetectorWithInputs(List.of(input)); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); @@ -209,7 +212,7 @@ public void testReplaceAggregationRuleWithDocRule_verifyFindings_success() throw "}"; SearchResponse response = executeSearchAndGetResponse(Rule.CUSTOM_RULES_INDEX, request, true); - assertEquals(1, response.getHits().getTotalHits().value); + assertEquals("Number of custom rules not correct",1, response.getHits().getTotalHits().value); request = "{\n" + " \"query\" : {\n" + @@ -226,13 +229,13 @@ public void testReplaceAggregationRuleWithDocRule_verifyFindings_success() throw String monitorType = ((Map) entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + monitorId))).get("monitor")).get("monitor_type"); - assertEquals(MonitorType.BUCKET_LEVEL_MONITOR.getValue(), monitorType); + assertEquals("Monitor type not correct", MonitorType.BUCKET_LEVEL_MONITOR.getValue(), monitorType); // Create random doc rule and 5 pre-packed rules and assign to detector - String randomDocRuleId = createRule(randomRule()); + String randomDocRuleId = createRule(randomRule(), "test_windows"); List prepackagedRules = getRandomPrePackagedRules(); input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(randomDocRuleId)), - prepackagedRules.stream().map(DetectorRule::new).collect(Collectors.toList())); + prepackagedRules.stream().map(DetectorRule::new).collect(Collectors.toList()), Collections.emptyList()); Detector updatedDetector = randomDetectorWithInputs(List.of(input)); Response updateResponse = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId, Collections.emptyMap(), toHttpEntity(updatedDetector)); @@ -248,7 +251,7 @@ public void testReplaceAggregationRuleWithDocRule_verifyFindings_success() throw detectorAsMap = (Map) hit.getSourceAsMap().get("detector"); List monitorIds = ((List) (detectorAsMap).get("monitor_id")); - assertEquals(1, monitorIds.size()); + assertEquals("Number of monitors not correct",1, monitorIds.size()); monitorId = monitorIds.get(0); monitorType = ((Map) entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + monitorId))).get("monitor")).get("monitor_type"); @@ -264,7 +267,7 @@ public void testReplaceAggregationRuleWithDocRule_verifyFindings_success() throw "}"; response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); - assertEquals(6, response.getHits().getTotalHits().value); + assertEquals("Number of rules on query index not correct",6, response.getHits().getTotalHits().value); // Verify findings indexDoc(index, "1", randomDoc(2, 5, "Info")); @@ -274,7 +277,7 @@ public void testReplaceAggregationRuleWithDocRule_verifyFindings_success() throw Map executeResults = entityAsMap(executeResponse); int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); // 5 prepackaged and 1 custom doc level rule - assertEquals(6, noOfSigmaRuleMatches); + assertEquals("Number of doc level sigma rules not correct",6, noOfSigmaRuleMatches); Map params = new HashMap<>(); params.put("detector_id", detectorId); @@ -283,7 +286,7 @@ public void testReplaceAggregationRuleWithDocRule_verifyFindings_success() throw assertNotNull(getFindingsBody); // When doc level monitor is being applied one finding is generated per document - assertEquals(2, getFindingsBody.get("total_findings")); + assertEquals("Number of total findings not correct", 2, getFindingsBody.get("total_findings")); Set docRuleIds = new HashSet<>(prepackagedRules); docRuleIds.add(randomDocRuleId); @@ -294,13 +297,13 @@ public void testReplaceAggregationRuleWithDocRule_verifyFindings_success() throw Set aggRulesFinding = ((List>)finding.get("queries")).stream().map(it -> it.get("id").toString()).collect( Collectors.toSet()); - assertTrue(docRuleIds.containsAll(aggRulesFinding)); + assertTrue("Finding rules not correct", docRuleIds.containsAll(aggRulesFinding)); List findingDocs = (List)finding.get("related_doc_ids"); - Assert.assertEquals(1, findingDocs.size()); + Assert.assertEquals("Number of documents not correct",1, findingDocs.size()); foundDocIds.addAll(findingDocs); } - assertTrue(Arrays.asList("1", "2").containsAll(foundDocIds)); + assertTrue("List of documents not correct", Arrays.asList("1", "2").containsAll(foundDocIds)); } /** @@ -343,7 +346,7 @@ public void testRemoveAllRulesAndUpdateDetector_success() throws IOException { "}"; SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); - assertEquals(randomPrepackagedRules.size(), response.getHits().getTotalHits().value); + assertEquals("Number of prepackaged rules not correct", randomPrepackagedRules.size(), response.getHits().getTotalHits().value); request = "{\n" + " \"query\" : {\n" + @@ -358,7 +361,7 @@ public void testRemoveAllRulesAndUpdateDetector_success() throws IOException { Map detectorAsMap = (Map) hit.getSourceAsMap().get("detector"); List monitorIds = ((List) (detectorAsMap).get("monitor_id")); - assertEquals(1, monitorIds.size()); + assertEquals("Number of monitors not correct", 1, monitorIds.size()); String monitorId = monitorIds.get(0); String monitorType = ((Map) entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + monitorId))).get("monitor")).get("monitor_type"); @@ -370,14 +373,11 @@ public void testRemoveAllRulesAndUpdateDetector_success() throws IOException { assertEquals("Update detector failed", RestStatus.OK, restStatus(updateResponse)); - Map updateResponseBody = asMap(updateResponse); - detectorId = updateResponseBody.get("_id").toString(); - hits = executeSearch(Detector.DETECTORS_INDEX, request); hit = hits.get(0); detectorAsMap = (Map) hit.getSourceAsMap().get("detector"); - assertTrue(((List) (detectorAsMap).get("monitor_id")).isEmpty()); + assertTrue("Monitor list not empty", ((List) (detectorAsMap).get("monitor_id")).isEmpty()); } /** @@ -406,10 +406,10 @@ public void testAddNewAggregationRule_verifyFindings_success() throws IOExceptio assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode()); - String sumRuleId = createRule(randomAggregationRule("sum", " > 1")); + String sumRuleId = createRule(randomAggregationRule("sum", " > 1"), "test_windows"); List detectorRules = List.of(new DetectorRule(sumRuleId)); DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), detectorRules, - Collections.emptyList()); + Collections.emptyList(), Collections.emptyList()); Detector detector = randomDetectorWithInputs(List.of(input)); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); @@ -430,12 +430,12 @@ public void testAddNewAggregationRule_verifyFindings_success() throws IOExceptio Map detectorMap = (HashMap)(hit.getSourceAsMap().get("detector")); List inputArr = detectorMap.get("inputs"); - assertEquals(1, ((Map>) inputArr.get(0)).get("detector_input").get("custom_rules").size()); + assertEquals("Number of custom rules not correct", 1, ((Map>) inputArr.get(0)).get("detector_input").get("custom_rules").size()); // Test adding the new max monitor and updating the existing sum monitor - String maxRuleId = createRule(randomAggregationRule("max", " > 3")); + String maxRuleId = createRule(randomAggregationRule("max", " > 3"), "test_windows"); DetectorInput newInput = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(maxRuleId), new DetectorRule(sumRuleId)), - Collections.emptyList()); + Collections.emptyList(), Collections.emptyList()); Detector updatedDetector = randomDetectorWithInputs(List.of(newInput)); Response updateResponse = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId, Collections.emptyMap(), toHttpEntity(updatedDetector)); @@ -457,7 +457,9 @@ public void testAddNewAggregationRule_verifyFindings_success() throws IOExceptio for(String monitorId: monitorIds) { Map monitor = (Map)(entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + monitorId)))).get("monitor"); - assertEquals(MonitorType.BUCKET_LEVEL_MONITOR.getValue(), monitor.get("monitor_type")); + + assertEquals("Invalid monitor type", MonitorType.BUCKET_LEVEL_MONITOR.getValue(), monitor.get("monitor_type")); + executeAlertingMonitor(monitorId, Collections.emptyMap()); } @@ -479,8 +481,8 @@ public void testAddNewAggregationRule_verifyFindings_success() throws IOExceptio List findingDocs = ((List) finding.get("related_doc_ids")); - assertEquals(2, findingDocs.size()); - assertTrue(Arrays.asList("1", "2").containsAll(findingDocs)); + assertEquals("Number of found documents not correct", 2, findingDocs.size()); + assertTrue("Wrong found doc ids", Arrays.asList("1", "2").containsAll(findingDocs)); String findingDetectorId = ((Map)((List) getFindingsBody.get("findings")).get(0)).get("detectorId").toString(); assertEquals(detectorId, findingDetectorId); @@ -514,14 +516,14 @@ public void testDeleteAggregationRule_verifyFindings_success() throws IOExceptio assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode()); List aggRuleIds = new ArrayList<>(); - String avgRuleId = createRule(randomAggregationRule("avg", " > 1")); + String avgRuleId = createRule(randomAggregationRule("avg", " > 1"), "test_windows"); aggRuleIds.add(avgRuleId); - String countRuleId = createRule(randomAggregationRule("count", " > 1")); + String countRuleId = createRule(randomAggregationRule("count", " > 1"), "test_windows"); aggRuleIds.add(countRuleId); List detectorRules = aggRuleIds.stream().map(DetectorRule::new).collect(Collectors.toList()); DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), detectorRules, - Collections.emptyList()); + Collections.emptyList(), Collections.emptyList()); Detector detector = randomDetectorWithInputs(List.of(input)); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); @@ -542,11 +544,11 @@ public void testDeleteAggregationRule_verifyFindings_success() throws IOExceptio Map detectorMap = (HashMap)(hit.getSourceAsMap().get("detector")); List inputArr = detectorMap.get("inputs"); - assertEquals(2, ((Map>) inputArr.get(0)).get("detector_input").get("custom_rules").size()); + assertEquals("Number of custom rules not correct", 2, ((Map>) inputArr.get(0)).get("detector_input").get("custom_rules").size()); // Test deleting the aggregation rule DetectorInput newInput = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(avgRuleId)), - Collections.emptyList()); + Collections.emptyList(), Collections.emptyList()); detector = randomDetectorWithInputs(List.of(newInput)); Response updateResponse = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId, Collections.emptyMap(), toHttpEntity(detector)); @@ -557,20 +559,16 @@ public void testDeleteAggregationRule_verifyFindings_success() throws IOExceptio Map updatedDetectorMap = (HashMap)(hit.getSourceAsMap().get("detector")); inputArr = updatedDetectorMap.get("inputs"); - assertEquals(1, ((Map>) inputArr.get(0)).get("detector_input").get("custom_rules").size()); - - inputArr = updatedDetectorMap.get("inputs"); - - assertEquals(1, ((Map>) inputArr.get(0)).get("detector_input").get("custom_rules").size()); + assertEquals("Number of custom rules not correct", 1, ((Map>) inputArr.get(0)).get("detector_input").get("custom_rules").size()); // Verify monitors List monitorIds = ((List) (updatedDetectorMap).get("monitor_id")); - assertEquals(1, monitorIds.size()); + assertEquals("Number of monitors not correct", 1, monitorIds.size()); Map monitor = (Map)(entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + monitorIds.get(0))))).get("monitor"); - assertEquals(MonitorType.BUCKET_LEVEL_MONITOR.getValue(), monitor.get("monitor_type")); + assertEquals("Invalid monitor type", MonitorType.BUCKET_LEVEL_MONITOR.getValue(), monitor.get("monitor_type")); indexDoc(index, "1", randomDoc(2, 4, "Info")); indexDoc(index, "2", randomDoc(3, 4, "Info")); @@ -584,7 +582,7 @@ public void testDeleteAggregationRule_verifyFindings_success() throws IOExceptio assertNotNull(getFindingsBody); - assertEquals(1, getFindingsBody.get("total_findings")); + assertEquals("Number of total findings not correct", 1, getFindingsBody.get("total_findings")); Map finding = ((List) getFindingsBody.get("findings")).get(0); Set aggRulesFinding = ((List>) finding.get("queries")).stream().map(it -> it.get("id").toString()).collect( @@ -594,8 +592,8 @@ public void testDeleteAggregationRule_verifyFindings_success() throws IOExceptio List findingDocs = (List) finding.get("related_doc_ids"); // Matches two findings because of the opCode rule uses (Info) - assertEquals(2, findingDocs.size()); - assertTrue(Arrays.asList("1", "2").containsAll(findingDocs)); + assertEquals("Number of found documents not correct", 2, findingDocs.size()); + assertTrue("Wrong found doc ids", Arrays.asList("1", "2").containsAll(findingDocs)); String findingDetectorId = ((Map)((List)getFindingsBody.get("findings")).get(0)).get("detectorId").toString(); assertEquals(detectorId, findingDetectorId); @@ -628,16 +626,16 @@ public void testReplaceAggregationRule_verifyFindings_success() throws IOExcepti assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode()); List aggRuleIds = new ArrayList<>(); - String avgRuleId = createRule(randomAggregationRule("avg", " > 1")); + String avgRuleId = createRule(randomAggregationRule("avg", " > 1"), "test_windows"); aggRuleIds.add(avgRuleId); - String minRuleId = createRule(randomAggregationRule("min", " > 1")); + String minRuleId = createRule(randomAggregationRule("min", " > 1"), "test_windows"); aggRuleIds.add(minRuleId); List detectorRules = aggRuleIds.stream().map(DetectorRule::new).collect(Collectors.toList()); List prepackagedDocRules = getRandomPrePackagedRules(); DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), detectorRules, - prepackagedDocRules.stream().map(DetectorRule::new).collect(Collectors.toList())); + prepackagedDocRules.stream().map(DetectorRule::new).collect(Collectors.toList()), Collections.emptyList()); Detector detector = randomDetectorWithInputs(List.of(input)); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); @@ -658,12 +656,12 @@ public void testReplaceAggregationRule_verifyFindings_success() throws IOExcepti Map detectorMap = (HashMap)(hit.getSourceAsMap().get("detector")); List inputArr = detectorMap.get("inputs"); - assertEquals(2, ((Map>) inputArr.get(0)).get("detector_input").get("custom_rules").size()); + assertEquals("Number of custom rules not correct", 2, ((Map>) inputArr.get(0)).get("detector_input").get("custom_rules").size()); - String maxRuleId = createRule(randomAggregationRule("max", " > 2")); + String maxRuleId = createRule(randomAggregationRule("max", " > 2"), "test_windows"); DetectorInput newInput = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(avgRuleId), new DetectorRule(maxRuleId)), - getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList())); + getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList()), Collections.emptyList()); detector = randomDetectorWithInputs(List.of(newInput)); createResponse = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId, Collections.emptyMap(), toHttpEntity(detector)); @@ -674,11 +672,11 @@ public void testReplaceAggregationRule_verifyFindings_success() throws IOExcepti Map updatedDetectorMap = (HashMap)(hit.getSourceAsMap().get("detector")); inputArr = updatedDetectorMap.get("inputs"); - assertEquals(2, ((Map>) inputArr.get(0)).get("detector_input").get("custom_rules").size()); + assertEquals("Number of custom rules not correct", 2, ((Map>) inputArr.get(0)).get("detector_input").get("custom_rules").size()); List monitorIds = ((List) (updatedDetectorMap).get("monitor_id")); - assertEquals(3, monitorIds.size()); + assertEquals("Number of monitors not correct", 3, monitorIds.size()); indexDoc(index, "1", randomDoc(2, 4, "Info")); indexDoc(index, "2", randomDoc(3, 4, "Info")); @@ -690,8 +688,8 @@ public void testReplaceAggregationRule_verifyFindings_success() throws IOExcepti executeAlertingMonitor(monitorId, Collections.emptyMap()); } - assertEquals(2, numberOfMonitorTypes.get(MonitorType.BUCKET_LEVEL_MONITOR.getValue()).intValue()); - assertEquals(1, numberOfMonitorTypes.get(MonitorType.DOC_LEVEL_MONITOR.getValue()).intValue()); + assertEquals("Number of bucket level monitors not correct", 2, numberOfMonitorTypes.get(MonitorType.BUCKET_LEVEL_MONITOR.getValue()).intValue()); + assertEquals("Number of doc level monitors not correct", 1, numberOfMonitorTypes.get(MonitorType.DOC_LEVEL_MONITOR.getValue()).intValue()); // Verify findings Map params = new HashMap<>(); params.put("detector_id", detectorId); @@ -699,7 +697,7 @@ public void testReplaceAggregationRule_verifyFindings_success() throws IOExcepti Map getFindingsBody = entityAsMap(getFindingsResponse); assertNotNull(getFindingsBody); - assertEquals(5, getFindingsBody.get("total_findings")); + assertEquals("Number of total findings not correct", 5, getFindingsBody.get("total_findings")); String findingDetectorId = ((Map)((List)getFindingsBody.get("findings")).get(0)).get("detectorId").toString(); assertEquals(detectorId, findingDetectorId); @@ -716,18 +714,17 @@ public void testReplaceAggregationRule_verifyFindings_success() throws IOExcepti List> queries = (List>)finding.get("queries"); Set findingRules = queries.stream().map(it -> it.get("id").toString()).collect(Collectors.toSet()); // In this test case all doc level rules are matching the finding rule ids - if(docLevelRules.containsAll(findingRules)) { + if (docLevelRules.containsAll(findingRules)) { docLevelFinding.addAll((List)finding.get("related_doc_ids")); } else { - String aggRuleId = findingRules.iterator().next(); - List findingDocs = (List)finding.get("related_doc_ids"); - Assert.assertEquals(2, findingDocs.size()); - assertTrue(Arrays.asList("1", "2").containsAll(findingDocs)); + + assertEquals("Number of found documents not correct", 2, findingDocs.size()); + assertTrue("Wrong found doc ids", Arrays.asList("1", "2").containsAll(findingDocs)); } } // Verify doc level finding - assertTrue(Arrays.asList("1", "2", "3").containsAll(docLevelFinding)); + assertTrue("Wrong found doc ids", Arrays.asList("1", "2", "3").containsAll(docLevelFinding)); } public void testMinAggregationRule_findingSuccess() throws IOException { @@ -747,10 +744,10 @@ public void testMinAggregationRule_findingSuccess() throws IOException { List aggRuleIds = new ArrayList<>(); String testOpCode = "Test"; - aggRuleIds.add(createRule(randomAggregationRule("min", " > 3", testOpCode))); + aggRuleIds.add(createRule(randomAggregationRule("min", " > 3", testOpCode), "test_windows")); List detectorRules = aggRuleIds.stream().map(DetectorRule::new).collect(Collectors.toList()); DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), detectorRules, - Collections.emptyList()); + Collections.emptyList(), Collections.emptyList()); Detector detector = randomDetectorWithInputs(List.of(input)); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); @@ -796,8 +793,9 @@ public void testMinAggregationRule_findingSuccess() throws IOException { List> findings = (List)getFindingsBody.get("findings"); for (Map finding : findings) { List findingDocs = (List)finding.get("related_doc_ids"); - Assert.assertEquals(1, findingDocs.size()); - assertTrue(Arrays.asList("7").containsAll(findingDocs)); + + assertEquals("Number of found documents not correct", 1, findingDocs.size()); + assertTrue("Wrong found doc ids", Arrays.asList("7").containsAll(findingDocs)); } String findingDetectorId = ((Map)((List)getFindingsBody.get("findings")).get(0)).get("detectorId").toString(); @@ -837,23 +835,26 @@ public void testMultipleAggregationAndDocRules_findingSuccess() throws IOExcepti String testOpCode = "Test"; // 5 custom aggregation rules - String sumRuleId = createRule(randomAggregationRule("sum", " > 1", infoOpCode)); - String maxRuleId = createRule(randomAggregationRule("max", " > 3", testOpCode)); - String minRuleId = createRule(randomAggregationRule("min", " > 3", testOpCode)); - String avgRuleId = createRule(randomAggregationRule("avg", " > 3", infoOpCode)); - String cntRuleId = createRule(randomAggregationRule("count", " > 3", "randomTestCode")); + String sumRuleId = createRule(randomAggregationRule("sum", " > 1", infoOpCode), "test_windows"); + String maxRuleId = createRule(randomAggregationRule("max", " > 3", testOpCode), "test_windows"); + String minRuleId = createRule(randomAggregationRule("min", " > 3", testOpCode), "test_windows"); + String avgRuleId = createRule(randomAggregationRule("avg", " > 3", infoOpCode), "test_windows"); + String cntRuleId = createRule(randomAggregationRule("count", " > 3", "randomTestCode"), "test_windows"); List aggRuleIds = List.of(sumRuleId, maxRuleId); - String randomDocRuleId = createRule(randomRule()); + // 1 custom doc level rule + String randomDocRuleId = createRule(randomRule(), "test_windows"); + // 5 prepackaged rules List prepackagedRules = getRandomPrePackagedRules(); List detectorRules = List.of(new DetectorRule(sumRuleId), new DetectorRule(maxRuleId), new DetectorRule(minRuleId), new DetectorRule(avgRuleId), new DetectorRule(cntRuleId), new DetectorRule(randomDocRuleId)); DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), detectorRules, - prepackagedRules.stream().map(DetectorRule::new).collect(Collectors.toList())); + prepackagedRules.stream().map(DetectorRule::new).collect(Collectors.toList()), Collections.emptyList()); Detector detector = randomDetectorWithInputs(List.of(input)); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); String request = "{\n" + " \"query\" : {\n" + @@ -863,9 +864,189 @@ public void testMultipleAggregationAndDocRules_findingSuccess() throws IOExcepti "}"; SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); - assertEquals(6, response.getHits().getTotalHits().value); + assertEquals("Number of doc level rules not correct", 6, response.getHits().getTotalHits().value); + + Map responseBody = asMap(createResponse); + String detectorId = responseBody.get("_id").toString(); + request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + Map updatedDetectorMap = (HashMap)(hit.getSourceAsMap().get("detector")); + List inputArr = updatedDetectorMap.get("inputs"); + + assertEquals("Number of custom rules not correct", 6, ((Map>) inputArr.get(0)).get("detector_input").get("custom_rules").size()); + + List monitorIds = ((List) (updatedDetectorMap).get("monitor_id")); + + assertEquals("Number of monitors not correct", 6, monitorIds.size()); + + indexDoc(index, "1", randomDoc(2, 4, infoOpCode)); + indexDoc(index, "2", randomDoc(3, 4, infoOpCode)); + indexDoc(index, "3", randomDoc(1, 4, infoOpCode)); + indexDoc(index, "4", randomDoc(5, 3, testOpCode)); + indexDoc(index, "5", randomDoc(2, 3, testOpCode)); + indexDoc(index, "6", randomDoc(4, 3, testOpCode)); + indexDoc(index, "7", randomDoc(6, 2, testOpCode)); + indexDoc(index, "8", randomDoc(1, 1, testOpCode)); + + Map numberOfMonitorTypes = new HashMap<>(); + + for (String monitorId: monitorIds) { + Map monitor = (Map)(entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + monitorId)))).get("monitor"); + numberOfMonitorTypes.merge(monitor.get("monitor_type"), 1, Integer::sum); + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + + // Assert monitor executions + Map executeResults = entityAsMap(executeResponse); + if (MonitorType.DOC_LEVEL_MONITOR.getValue().equals(monitor.get("monitor_type"))) { + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + // 5 prepackaged and 1 custom doc level rule + assertEquals("Number of sigma rules not correct", 6, noOfSigmaRuleMatches); + } else { + for(String ruleId: aggRuleIds) { + Object rule = (((Map)((Map)((List)((Map)executeResults.get("input_results")).get("results")).get(0)).get("aggregations")).get(ruleId)); + if (rule != null) { + if (ruleId.equals(sumRuleId)) { + assertRuleMonitorFinding(executeResults, ruleId,3, List.of("4")); + } else if (ruleId.equals(maxRuleId)) { + assertRuleMonitorFinding(executeResults, ruleId,5, List.of("2", "3")); + } + else if (ruleId.equals(minRuleId)) { + assertRuleMonitorFinding(executeResults, ruleId,1, List.of("2")); + } + } + } + } + } + + assertEquals("Number of bucket level monitors not correct", 5, numberOfMonitorTypes.get(MonitorType.BUCKET_LEVEL_MONITOR.getValue()).intValue()); + assertEquals("Number of doc level monitors not correct", 1, numberOfMonitorTypes.get(MonitorType.DOC_LEVEL_MONITOR.getValue()).intValue()); + + Map params = new HashMap<>(); + params.put("detector_id", detectorId); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + + // Assert findings + assertNotNull(getFindingsBody); + // 8 findings from doc level rules, and 3 findings for aggregation (sum, max and min) + assertEquals("Number of total findings not correct", 11, getFindingsBody.get("total_findings")); + + String findingDetectorId = ((Map)((List)getFindingsBody.get("findings")).get(0)).get("detectorId").toString(); + assertEquals(detectorId, findingDetectorId); + + String findingIndex = ((Map)((List)getFindingsBody.get("findings")).get(0)).get("index").toString(); + assertEquals(index, findingIndex); + + List docLevelFinding = new ArrayList<>(); + List> findings = (List) getFindingsBody.get("findings"); + + Set docLevelRules = new HashSet<>(prepackagedRules); + docLevelRules.add(randomDocRuleId); + + for(Map finding : findings) { + List> queries = (List>)finding.get("queries"); + Set findingRuleIds = queries.stream().map(it -> it.get("id").toString()).collect(Collectors.toSet()); + // Doc level finding matches all doc level rules (including the custom one) in this test case + if (docLevelRules.containsAll(findingRuleIds)) { + docLevelFinding.addAll((List)finding.get("related_doc_ids")); + } else { + // In the case of bucket level monitors, queries will always contain one value + String aggRuleId = findingRuleIds.iterator().next(); + List findingDocs = (List)finding.get("related_doc_ids"); + + if (aggRuleId.equals(sumRuleId)) { + assertTrue("Wrong found doc ids for sum rule", List.of("1", "2", "3").containsAll(findingDocs)); + } else if (aggRuleId.equals(maxRuleId)) { + assertTrue("Wrong found doc ids for max rule", List.of("4", "5", "6", "7").containsAll(findingDocs)); + } else if (aggRuleId.equals(minRuleId)) { + assertTrue("Wrong found doc ids for min rule", List.of("7").containsAll(findingDocs)); + } + } + } + + assertTrue(Arrays.asList("Wrong found doc ids", "1", "2", "3", "4", "5", "6", "7", "8").containsAll(docLevelFinding)); + } + + /** + * 1. Creates detector with aggregation and prepackaged rules; + * aggregation rules - windows category; custom doc level rule - windows category; prepackaged rules - test_windows category + * 2. Verifies monitor execution + * 3. Verifies findings by getting the findings by detector id (join findings for all rule categories/ log types) + * @throws IOException + */ + public void testMultipleAggregationAndDocRulesForMultipleDetectorTypes_findingSuccess() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response createMappingResponse = client().performRequest(createMappingRequest); + + assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode()); + + String lin = "s3"; + + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + lin + "\", " + + " \"partial\":true" + + "}" + ); + + createMappingResponse = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode()); + + String infoOpCode = "Info"; + String testOpCode = "Test"; + + // 5 custom aggregation rules + String sumRuleId = createRule(randomAggregationRule("sum", " > 1", infoOpCode), windowsDetectorType()); + String maxRuleId = createRule(randomAggregationRule("max", " > 3", testOpCode), windowsDetectorType()); + String minRuleId = createRule(randomAggregationRule("min", " > 3", testOpCode), windowsDetectorType()); + String avgRuleId = createRule(randomAggregationRule("avg", " > 3", infoOpCode), windowsDetectorType()); + String cntRuleId = createRule(randomAggregationRule("count", " > 3", "randomTestCode"), windowsDetectorType()); + + List aggRuleIds = List.of(sumRuleId, maxRuleId); + String randomDocRuleId = createRule(randomRule(), windowsDetectorType()); + List prepackagedRules = getRandomPrePackagedRules(); + + List detectorRules = List.of(new DetectorRule(sumRuleId), new DetectorRule(maxRuleId), new DetectorRule(minRuleId), + new DetectorRule(avgRuleId), new DetectorRule(cntRuleId), new DetectorRule(randomDocRuleId)); + + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), detectorRules, + prepackagedRules.stream().map(DetectorRule::new).collect(Collectors.toList()), List.of(DetectorType.TEST_WINDOWS, DetectorType.WINDOWS)); + Detector detector = randomDetectorWithInputs(List.of(input)); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + assertEquals("Number of doc level rules not correct for test_windows rule category", 5, response.getHits().getTotalHits().value); + + response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(windowsDetectorType()), request, true); + assertEquals("Number of doc level rules not correct for windows rule category", 1, response.getHits().getTotalHits().value); + Map responseBody = asMap(createResponse); String detectorId = responseBody.get("_id").toString(); request = "{\n" + @@ -880,11 +1061,11 @@ public void testMultipleAggregationAndDocRules_findingSuccess() throws IOExcepti Map updatedDetectorMap = (HashMap)(hit.getSourceAsMap().get("detector")); List inputArr = updatedDetectorMap.get("inputs"); - assertEquals(6, ((Map>) inputArr.get(0)).get("detector_input").get("custom_rules").size()); + assertEquals("Number of custom rules not correct", 6, ((Map>) inputArr.get(0)).get("detector_input").get("custom_rules").size()); List monitorIds = ((List) (updatedDetectorMap).get("monitor_id")); - assertEquals(6, monitorIds.size()); + assertEquals(7, monitorIds.size()); indexDoc(index, "1", randomDoc(2, 4, infoOpCode)); indexDoc(index, "2", randomDoc(3, 4, infoOpCode)); @@ -897,6 +1078,8 @@ public void testMultipleAggregationAndDocRules_findingSuccess() throws IOExcepti Map numberOfMonitorTypes = new HashMap<>(); + String windowsMonitorId = (String)((Map)updatedDetectorMap.get("doc_monitor_id_per_category")).get("windows"); + String testWindowsMonitorId = (String)((Map)updatedDetectorMap.get("doc_monitor_id_per_category")).get("test_windows"); for (String monitorId: monitorIds) { Map monitor = (Map)(entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + monitorId)))).get("monitor"); numberOfMonitorTypes.merge(monitor.get("monitor_type"), 1, Integer::sum); @@ -907,12 +1090,18 @@ public void testMultipleAggregationAndDocRules_findingSuccess() throws IOExcepti if (MonitorType.DOC_LEVEL_MONITOR.getValue().equals(monitor.get("monitor_type"))) { int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); // 5 prepackaged and 1 custom doc level rule - assertEquals(6, noOfSigmaRuleMatches); + if (monitorId.equals(windowsMonitorId)) { + assertEquals("Number of doc level rules in monitor executions for windows monitor not correct", 1, noOfSigmaRuleMatches); + } + if (monitorId.equals(testWindowsMonitorId)) { + assertEquals("Number of doc level rules in monitor executions for test_windows monitor not correct", 5, noOfSigmaRuleMatches); + } + } else { for(String ruleId: aggRuleIds) { Object rule = (((Map)((Map)((List)((Map)executeResults.get("input_results")).get("results")).get(0)).get("aggregations")).get(ruleId)); - if(rule != null) { - if(ruleId == sumRuleId) { + if (rule != null) { + if (ruleId == sumRuleId) { assertRuleMonitorFinding(executeResults, ruleId,3, List.of("4")); } else if (ruleId == maxRuleId) { assertRuleMonitorFinding(executeResults, ruleId,5, List.of("2", "3")); @@ -925,8 +1114,8 @@ else if (ruleId == minRuleId) { } } - assertEquals(5, numberOfMonitorTypes.get(MonitorType.BUCKET_LEVEL_MONITOR.getValue()).intValue()); - assertEquals(1, numberOfMonitorTypes.get(MonitorType.DOC_LEVEL_MONITOR.getValue()).intValue()); + assertEquals("Number of bucket level monitors not correct", 5, numberOfMonitorTypes.get(MonitorType.BUCKET_LEVEL_MONITOR.getValue()).intValue()); + assertEquals("Number of doc level monitors not correct", 2, numberOfMonitorTypes.get(MonitorType.DOC_LEVEL_MONITOR.getValue()).intValue()); Map params = new HashMap<>(); params.put("detector_id", detectorId); @@ -935,8 +1124,10 @@ else if (ruleId == minRuleId) { // Assert findings assertNotNull(getFindingsBody); - // 8 findings from doc level rules, and 3 findings for aggregation (sum, max and min) - assertEquals(11, getFindingsBody.get("total_findings")); + // 8 findings for 5 prepackaged doc level rules for test_windows category + // 8 findings for 1 custom doc level rule for windows category + // 3 findings for bucket level monitors for windows + assertEquals("Number of total findings not correct",19, getFindingsBody.get("total_findings")); String findingDetectorId = ((Map)((List)getFindingsBody.get("findings")).get(0)).get("detectorId").toString(); assertEquals(detectorId, findingDetectorId); @@ -954,32 +1145,271 @@ else if (ruleId == minRuleId) { List> queries = (List>)finding.get("queries"); Set findingRuleIds = queries.stream().map(it -> it.get("id").toString()).collect(Collectors.toSet()); // Doc level finding matches all doc level rules (including the custom one) in this test case - if(docLevelRules.containsAll(findingRuleIds)) { + if (docLevelRules.containsAll(findingRuleIds)) { docLevelFinding.addAll((List)finding.get("related_doc_ids")); } else { // In the case of bucket level monitors, queries will always contain one value String aggRuleId = findingRuleIds.iterator().next(); List findingDocs = (List)finding.get("related_doc_ids"); - if(aggRuleId.equals(sumRuleId)) { - assertTrue(List.of("1", "2", "3").containsAll(findingDocs)); - } else if(aggRuleId.equals(maxRuleId)) { - assertTrue(List.of("4", "5", "6", "7").containsAll(findingDocs)); - } else if(aggRuleId.equals( minRuleId)) { - assertTrue(List.of("7").containsAll(findingDocs)); + if (aggRuleId.equals(sumRuleId)) { + assertTrue("Wrong found doc ids for sum rule", List.of("1", "2", "3").containsAll(findingDocs)); + } else if (aggRuleId.equals(maxRuleId)) { + assertTrue("Wrong found doc ids for max rule", List.of("4", "5", "6", "7").containsAll(findingDocs)); + } else if (aggRuleId.equals( minRuleId)) { + assertTrue("Wrong found doc ids for min rule", List.of("7").containsAll(findingDocs)); } } } - assertTrue(Arrays.asList("1", "2", "3", "4", "5", "6", "7", "8").containsAll(docLevelFinding)); + assertTrue("Wrong found doc ids", Arrays.asList("1", "2", "3", "4", "5", "6", "7", "8").containsAll(docLevelFinding)); + } + + /** + * 1. Create aggregation rules - windows category; pre-packaged rules - test_windows category; random doc rule - windows category + * 2. Verifies monitor number and rule types and their numbers + * 3. Updates the detector by removing the custom doc level rule and all prepackaged rules + * 4. Verifies that two query indices are removed + * 5. Verifies removed monitors + * @throws IOException + */ + public void testRemoveDocLevelRulesAndOneDetectorType_findingSuccess() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response createMappingResponse = client().performRequest(createMappingRequest); + + assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode()); + + String infoOpCode = "Info"; + String testOpCode = "Test"; + + // 5 custom aggregation rules + String sumRuleId = createRule(randomAggregationRule("sum", " > 1", infoOpCode), windowsDetectorType()); + String maxRuleId = createRule(randomAggregationRule("max", " > 3", testOpCode), windowsDetectorType()); + String minRuleId = createRule(randomAggregationRule("min", " > 3", testOpCode), windowsDetectorType()); + String avgRuleId = createRule(randomAggregationRule("avg", " > 3", infoOpCode), windowsDetectorType()); + String cntRuleId = createRule(randomAggregationRule("count", " > 3", "randomTestCode"), windowsDetectorType()); + String randomDocRuleId = createRule(randomRule(), windowsDetectorType()); + List prepackagedRules = getRandomPrePackagedRules(); + + List detectorRules = List.of(new DetectorRule(sumRuleId), new DetectorRule(maxRuleId), new DetectorRule(minRuleId), + new DetectorRule(avgRuleId), new DetectorRule(cntRuleId), new DetectorRule(randomDocRuleId)); + + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), detectorRules, + prepackagedRules.stream().map(DetectorRule::new).collect(Collectors.toList()), List.of(DetectorType.TEST_WINDOWS, DetectorType.WINDOWS)); + Detector detector = randomDetectorWithInputs(List.of(input)); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex("test_windows"), request, true); + assertEquals("Number of doc level rules for test_windows category not correct", 5, response.getHits().getTotalHits().value); + + response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex("windows"), request, true); + assertEquals("Number of doc level rules for windows category not correct", 1, response.getHits().getTotalHits().value); + + response = executeSearchAndGetResponse(Rule.CUSTOM_RULES_INDEX, request, true); + assertEquals("Number of custom rules not correct", 6, response.getHits().getTotalHits().value); + + + Map responseBody = asMap(createResponse); + String detectorId = responseBody.get("_id").toString(); + request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + Map detectorMap = (HashMap)(hit.getSourceAsMap().get("detector")); + List inputArr = detectorMap.get("inputs"); + + assertEquals("Number of custom rules not correct", 6, ((Map>) inputArr.get(0)).get("detector_input").get("custom_rules").size()); + + List monitorIds = ((List) (detectorMap).get("monitor_id")); + + assertEquals("Number of monitors not correct", 7, monitorIds.size()); + Map docLevelMonitorIdPerCategory = ((Map)((Map)hit.getSourceAsMap().get("detector")).get("doc_monitor_id_per_category")); + Collection docLevelMonitorIds = docLevelMonitorIdPerCategory.values(); + // verify that detector list of doc monitor ids is correct + assertTrue("Monitor list doesn't contain doc level monitor ids", monitorIds.containsAll(docLevelMonitorIds)); + // Updating detector - removing prepackaged and custom doc level rules for test_windows category; Removing the detector type + detectorRules = List.of(new DetectorRule(sumRuleId), new DetectorRule(maxRuleId), new DetectorRule(minRuleId), new DetectorRule(avgRuleId), new DetectorRule(cntRuleId)); + input = new DetectorInput("windows detector for security analytics", List.of("windows"), detectorRules, Collections.emptyList(), List.of(DetectorType.WINDOWS)); + + Detector updatedDetector = randomDetectorWithInputs(List.of(input)); + Response updateResponse = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId, Collections.emptyMap(), toHttpEntity(updatedDetector)); + + assertEquals("Update detector failed", RestStatus.OK, restStatus(updateResponse)); + + request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + // Custom created doc rule removed from detector + response = executeSearchAndGetResponse(Rule.CUSTOM_RULES_INDEX, request, true); + assertEquals("Number of custom rules not correct after removal of query index", 6, response.getHits().getTotalHits().value); + + request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + hits = executeSearch(Detector.DETECTORS_INDEX, request); + hit = hits.get(0); + detectorMap = (HashMap)(hit.getSourceAsMap().get("detector")); + monitorIds = ((List) (detectorMap).get("monitor_id")); + // Verify that two doc level monitors are removed - one for windows (removed custom doc level rule) and the second for test_windows category + assertEquals("Number of monitors not correct after doc level monitors removed", 5, monitorIds.size()); + assertTrue("Removed doc level monitors still exists in monitor list", !monitorIds.containsAll(docLevelMonitorIds)); + } + + /** + * 1. Create pre-packaged rules - test_windows category; random doc rule - windows category and one aggregation rule + * 2. Verifies monitor number and rule types and their numbers + * 3. Updates the detector by removing the custom doc level rule and all prepackaged rules + * 4. Verifies that two query indices are removed + * 5. Verifies removed monitors + * @throws IOException + */ + public void testRemoveBucketLevelRuleAndOneDetectorType_findingSuccess() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response createMappingResponse = client().performRequest(createMappingRequest); + + assertEquals(HttpStatus.SC_OK, createMappingResponse.getStatusLine().getStatusCode()); + + String testOpCode = "Test"; + + // 1 custom aggregation rules + String maxRuleId = createRule(randomAggregationRule("max", " > 3", testOpCode)); + + String customDocRuleId = createRule(randomRule(), "test_windows"); + List prepackagedRules = getRandomPrePackagedRules(); + + List detectorRules = List.of(new DetectorRule(maxRuleId), new DetectorRule(customDocRuleId)); + + DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), detectorRules, + prepackagedRules.stream().map(DetectorRule::new).collect(Collectors.toList()), List.of(DetectorType.TEST_WINDOWS, DetectorType.WINDOWS)); + Detector detector = randomDetectorWithInputs(List.of(input)); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex("test_windows"), request, true); + assertEquals("Number of doc level rules not correct",6, response.getHits().getTotalHits().value); + + response = executeSearchAndGetResponse(Rule.CUSTOM_RULES_INDEX, request, true); + assertEquals("Number of custom rules not correct", 2, response.getHits().getTotalHits().value); + + Map responseBody = asMap(createResponse); + String detectorId = responseBody.get("_id").toString(); + request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + Map detectorMap = (HashMap)(hit.getSourceAsMap().get("detector")); + List inputArr = detectorMap.get("inputs"); + + assertEquals("Number of custom rules not correct", 2, ((Map>) inputArr.get(0)).get("detector_input").get("custom_rules").size()); + + List monitorIds = ((List) (detectorMap).get("monitor_id")); + + assertEquals("Number of monitors not correct", 2, monitorIds.size()); + + // verify that detector list of doc monitor ids is correct + Map docLevelMonitorIdPerCategory = ((Map)((Map)hit.getSourceAsMap().get("detector")).get("doc_monitor_id_per_category")); + Collection docLevelMonitorIds = docLevelMonitorIdPerCategory.values(); + assertTrue(monitorIds.containsAll(docLevelMonitorIds)); + + // verify that detector list of bucket monitor ids is correct + Map bucketLevelMonitorIdPerRule = ((Map)((Map)hit.getSourceAsMap().get("detector")).get("bucket_monitor_id_rule_id")); + Collection bucketLevelMonitorIds = bucketLevelMonitorIdPerRule.values(); + + assertTrue("Monitor list doesn't contain all bucket level monitors", monitorIds.containsAll(bucketLevelMonitorIds)); + + // Updating detector - removing prepackaged and custom doc level rules for test_windows category; Removing the detector type + detectorRules = List.of(new DetectorRule(customDocRuleId)); + input = new DetectorInput("windows detector for security analytics", List.of("windows"), detectorRules, Collections.emptyList(), List.of(DetectorType.WINDOWS)); + Detector updatedDetector = randomDetectorWithInputs(List.of(input)); + Response updateResponse = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId, Collections.emptyMap(), toHttpEntity(updatedDetector)); + assertEquals("Update detector failed", RestStatus.OK, restStatus(updateResponse)); + + request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + // Custom created doc rule removed from detector + response = executeSearchAndGetResponse(Rule.CUSTOM_RULES_INDEX, request, true); + assertEquals("Number of custom rules not correct after ", 2, response.getHits().getTotalHits().value); + + request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + hits = executeSearch(Detector.DETECTORS_INDEX, request); + hit = hits.get(0); + detectorMap = (HashMap)(hit.getSourceAsMap().get("detector")); + monitorIds = ((List) (detectorMap).get("monitor_id")); + + assertEquals("Number of monitors not correct after monitor removal of bucket level monitor", 1, monitorIds.size()); + assertTrue(!monitorIds.containsAll(bucketLevelMonitorIds)); } private static void assertRuleMonitorFinding(Map executeResults, String ruleId, int expectedDocCount, List expectedTriggerResult) { List> buckets = ((List>)(((Map)((Map)((Map)((List)((Map) executeResults.get("input_results")).get("results")).get(0)).get("aggregations")).get("result_agg")).get("buckets"))); Integer docCount = buckets.stream().mapToInt(it -> (Integer)it.get("doc_count")).sum(); - assertEquals(expectedDocCount, docCount.intValue()); + assertEquals("Total doc count not correct", expectedDocCount, docCount.intValue()); List triggerResultBucketKeys = ((Map)((Map) ((Map)executeResults.get("trigger_results")).get(ruleId)).get("agg_result_buckets")).keySet().stream().collect(Collectors.toList()); - assertEquals(expectedTriggerResult, triggerResultBucketKeys); + assertEquals("Trigger result not correct", expectedTriggerResult, triggerResultBucketKeys); } } diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java index 1711a6ca3..a23fbcfc5 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java @@ -76,8 +76,8 @@ public void testCreatingADetector() throws IOException { Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("findings_index")); Assert.assertFalse(((Map) responseBody.get("detector")).containsKey("alert_index")); - String detectorTypeInResponse = (String) ((Map)responseBody.get("detector")).get("detector_type"); - Assert.assertEquals("Detector type incorrect", randomDetectorType().toLowerCase(Locale.ROOT), detectorTypeInResponse); + List detectorTypesInResponse = ((List)((Map)((Map)((List)((Map) responseBody.get("detector")).get("inputs")).get(0)).get("detector_input")).get("detector_types")); + Assert.assertTrue("Detector type incorrect", detectorTypesInResponse.contains(randomDetectorType().toLowerCase(Locale.ROOT))); String request = "{\n" + " \"query\" : {\n" + @@ -126,7 +126,7 @@ public void testCreatingADetectorWithNonExistingCustomRule() throws IOException Response response = client().performRequest(createMappingRequest); assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(java.util.UUID.randomUUID().toString())), - getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList())); + getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList()), Collections.emptyList()); Detector detector = randomDetectorWithInputs(List.of(input)); try { @@ -213,8 +213,8 @@ public void testGettingADetector() throws IOException { Assert.assertEquals(createdId, responseBody.get("_id")); Assert.assertNotNull(responseBody.get("detector")); - String detectorTypeInResponse = (String) ((Map)responseBody.get("detector")).get("detector_type"); - Assert.assertEquals("Detector type incorrect", randomDetectorType().toLowerCase(Locale.ROOT), detectorTypeInResponse); + List detectorTypesInResponse = ((List)((Map)((Map)((List)((Map) responseBody.get("detector")).get("inputs")).get(0)).get("detector_input")).get("detector_types")); + Assert.assertTrue("Detector type incorrect", detectorTypesInResponse.contains(randomDetectorType().toLowerCase(Locale.ROOT))); } @SuppressWarnings("unchecked") @@ -254,8 +254,9 @@ public void testSearchingDetectors() throws IOException { List> hits = ((List>) ((Map) searchResponseBody.get("hits")).get("hits")); Map hit = hits.get(0); - String detectorTypeInResponse = (String) ((Map) hit.get("_source")).get("detector_type"); - Assert.assertEquals("Detector type incorrect", detectorTypeInResponse, randomDetectorType().toLowerCase(Locale.ROOT)); + + List detectorTypesInResponse = ((List)((Map)((Map)((List)((Map) hit.get("_source")).get("inputs")).get(0)).get("detector_input")).get("detector_types")); + Assert.assertTrue("Detector type incorrect", detectorTypesInResponse.contains(randomDetectorType().toLowerCase(Locale.ROOT))); } @SuppressWarnings("unchecked") @@ -285,7 +286,7 @@ public void testCreatingADetectorWithCustomRules() throws IOException { String createdId = responseBody.get("_id").toString(); DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(createdId)), - getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList())); + getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList()), List.of()); Detector detector = randomDetectorWithInputs(List.of(input)); createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); @@ -312,8 +313,8 @@ public void testCreatingADetectorWithCustomRules() throws IOException { List hits = executeSearch(Detector.DETECTORS_INDEX, request); SearchHit hit = hits.get(0); - String detectorType = (String) ((Map) hit.getSourceAsMap().get("detector")).get("detector_type"); - Assert.assertEquals("Detector type incorrect", detectorType, randomDetectorType().toLowerCase(Locale.ROOT)); + List detectorTypesInResponse = ((List)((Map)((Map)((List)((Map) responseBody.get("detector")).get("inputs")).get(0)).get("detector_input")).get("detector_types")); + Assert.assertTrue("Detector type incorrect", detectorTypesInResponse.contains(randomDetectorType().toLowerCase(Locale.ROOT))); String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); @@ -332,6 +333,7 @@ public void testCreatingADetectorWithAggregationRules() throws IOException { // Execute CreateMappingsAction to add alias mapping for index Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); // both req params and req body are supported + // TODO - was windows createMappingRequest.setJsonEntity( "{ \"index_name\":\"" + index + "\"," + " \"rule_topic\":\"test_windows\", " + @@ -342,10 +344,10 @@ public void testCreatingADetectorWithAggregationRules() throws IOException { Response response = client().performRequest(createMappingRequest); assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); - String customAvgRuleId = createRule(productIndexAvgAggRule()); + String customAvgRuleId = createRule(productIndexAvgAggRule(), "test_windows"); DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(customAvgRuleId)), - getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList())); + getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList()), List.of()); Detector detector = randomDetectorWithInputs(List.of(input)); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); @@ -384,7 +386,7 @@ public void testCreatingADetectorWithAggregationRules() throws IOException { String firstMonitorId = monitorIds.get(0); String firstMonitorType = ((Map) entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + firstMonitorId))).get("monitor")).get("monitor_type"); - if(MonitorType.BUCKET_LEVEL_MONITOR.getValue().equals(firstMonitorType)){ + if (MonitorType.BUCKET_LEVEL_MONITOR.getValue().equals(firstMonitorType)){ bucketLevelMonitorId = firstMonitorId; } monitorTypes.add(firstMonitorType); @@ -392,7 +394,7 @@ public void testCreatingADetectorWithAggregationRules() throws IOException { String secondMonitorId = monitorIds.get(1); String secondMonitorType = ((Map) entityAsMap(client().performRequest(new Request("GET", "/_plugins/_alerting/monitors/" + secondMonitorId))).get("monitor")).get("monitor_type"); monitorTypes.add(secondMonitorType); - if(MonitorType.BUCKET_LEVEL_MONITOR.getValue().equals(secondMonitorType)){ + if (MonitorType.BUCKET_LEVEL_MONITOR.getValue().equals(secondMonitorType)){ bucketLevelMonitorId = secondMonitorId; } Assert.assertTrue(Arrays.asList(MonitorType.BUCKET_LEVEL_MONITOR.getValue(), MonitorType.DOC_LEVEL_MONITOR.getValue()).containsAll(monitorTypes)); @@ -465,14 +467,18 @@ public void testUpdateADetector() throws IOException { String createdId = responseBody.get("_id").toString(); DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(createdId)), - getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList())); + getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList()), List.of()); Detector updatedDetector = randomDetectorWithInputs(List.of(input)); + Response updateResponse = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId, Collections.emptyMap(), toHttpEntity(updatedDetector)); + + responseBody = asMap(updateResponse); + Assert.assertEquals("Update detector failed", RestStatus.OK, restStatus(updateResponse)); - String detectorTypeInResponse = (String) ((Map) (asMap(updateResponse).get("detector"))).get("detector_type"); - Assert.assertEquals("Detector type incorrect", randomDetectorType().toLowerCase(Locale.ROOT), detectorTypeInResponse); + List detectorTypesInResponse = ((List)((Map)((Map)((List)((Map) responseBody.get("detector")).get("inputs")).get(0)).get("detector_input")).get("detector_types")); + Assert.assertTrue("Detector type incorrect", detectorTypesInResponse.contains(randomDetectorType().toLowerCase(Locale.ROOT))); request = "{\n" + " \"query\" : {\n" + @@ -498,7 +504,7 @@ public void testUpdateANonExistingDetector() throws IOException { ); DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(), - getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList())); + getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList()), Collections.emptyList()); Detector updatedDetector = randomDetectorWithInputs(List.of(input)); try { @@ -510,7 +516,7 @@ public void testUpdateANonExistingDetector() throws IOException { public void testUpdateADetectorWithIndexNotExists() throws IOException { DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(), - getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList())); + getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList()), Collections.emptyList()); Detector updatedDetector = randomDetectorWithInputs(List.of(input)); try { @@ -719,7 +725,7 @@ public void testCreatingADetectorWithTimestampFieldAliasMapping() throws IOExcep String customAvgRuleId = createRule(productIndexAvgAggRule()); DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(customAvgRuleId)), - getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList())); + getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList()), List.of()); Detector detector = randomDetectorWithInputs(List.of(input)); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); @@ -822,7 +828,7 @@ public void testCreatingADetectorWithTimestampFieldAliasMapping_verifyTimeRangeI String customAvgRuleId = createRule(productIndexAvgAggRule()); DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(customAvgRuleId)), - getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList())); + getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList()), List.of()); Detector detector = randomDetectorWithInputs(List.of(input)); Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java index ea4936a4a..c5304c980 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/RuleRestApiIT.java @@ -412,7 +412,7 @@ public void testUpdatingUnusedRuleAfterDetectorIndexCreated() throws IOException String createdId = responseBody.get("_id").toString(); DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(), - getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList())); + getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList()), Collections.emptyList()); Detector detector = randomDetectorWithInputs(List.of(input)); createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); @@ -451,7 +451,7 @@ public void testUpdatingUsedRule() throws IOException { String createdId = responseBody.get("_id").toString(); DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(createdId)), - getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList())); + getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList()), Collections.emptyList()); Detector detector = randomDetectorWithInputs(List.of(input)); createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); @@ -570,7 +570,7 @@ public void testDeletingUnusedRuleAfterDetectorIndexCreated() throws IOException String createdId = responseBody.get("_id").toString(); DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(), - getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList())); + getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList()), Collections.emptyList()); Detector detector = randomDetectorWithInputs(List.of(input)); createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); @@ -607,7 +607,7 @@ public void testDeletingUsedRule() throws IOException { String createdId = responseBody.get("_id").toString(); DetectorInput input = new DetectorInput("windows detector for security analytics", List.of("windows"), List.of(new DetectorRule(createdId)), - getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList())); + getRandomPrePackagedRules().stream().map(DetectorRule::new).collect(Collectors.toList()), Collections.emptyList()); Detector detector = randomDetectorWithInputs(List.of(input)); createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector));