From e44ccb216d946a386e8bccffff2fd7ad064036a8 Mon Sep 17 00:00:00 2001 From: Petar Dzepina Date: Wed, 9 Nov 2022 21:17:14 +0100 Subject: [PATCH] =?UTF-8?q?fixed=20GetFindings=20not=20searching=20all=20i?= =?UTF-8?q?ndices;=20fixed=20proper=20deletion=20of=E2=80=A6=20(#122)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fixed GetFindings not searching all indices; fixed proper deletion of old history indices Signed-off-by: Petar Dzepina (cherry picked from commit 8dd1c9ff5febc16f2b36b13cf0a3f4c5efa5c7c9) --- .../alerts/AlertsService.java | 4 +- .../monitors/DetectorMonitorConfig.java | 41 ++++- .../findings/FindingsService.java | 4 +- .../DetectorIndexManagementService.java | 43 ++--- .../SecurityAnalyticsRestTestCase.java | 2 +- .../securityanalytics/alerts/AlertsIT.java | 173 +++++++++++++++++- .../securityanalytics/findings/FindingIT.java | 94 +++++++++- 7 files changed, 327 insertions(+), 34 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/alerts/AlertsService.java b/src/main/java/org/opensearch/securityanalytics/alerts/AlertsService.java index 3d7f228e7..abad1a894 100644 --- a/src/main/java/org/opensearch/securityanalytics/alerts/AlertsService.java +++ b/src/main/java/org/opensearch/securityanalytics/alerts/AlertsService.java @@ -82,7 +82,7 @@ public void onResponse(GetDetectorResponse getDetectorResponse) { AlertsService.this.getAlertsByMonitorIds( monitorToDetectorMapping, monitorIds, - DetectorMonitorConfig.getAlertsIndex(detector.getDetectorType()), + DetectorMonitorConfig.getAllAlertsIndicesPattern(detector.getDetectorType()), table, severityLevel, alertState, @@ -193,7 +193,7 @@ public void getAlerts( AlertsService.this.getAlertsByMonitorIds( monitorToDetectorMapping, allMonitorIds, - DetectorMonitorConfig.getAlertsIndex(detectorType.getDetectorType()), + DetectorMonitorConfig.getAllAlertsIndicesPattern(detectorType.getDetectorType()), table, severityLevel, alertState, diff --git a/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java b/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java index c3129317c..f77ade3b5 100644 --- a/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java +++ b/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java @@ -4,8 +4,8 @@ */ package org.opensearch.securityanalytics.config.monitors; -import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; import org.opensearch.securityanalytics.model.Detector; import java.util.Arrays; @@ -18,9 +18,11 @@ public class DetectorMonitorConfig { public static final String OPENSEARCH_DEFAULT_RULE_INDEX = ".opensearch-sap-detectors-queries-default"; public static final String OPENSEARCH_DEFAULT_ALERT_INDEX = ".opensearch-sap-alerts-default"; + public static final String OPENSEARCH_DEFAULT_ALL_ALERT_INDICES_PATTERN = ".opensearch-sap-alerts-default*"; public static final String OPENSEARCH_DEFAULT_ALERT_HISTORY_INDEX = ".opensearch-sap-alerts-history-default"; public static final String OPENSEARCH_DEFAULT_ALERT_HISTORY_INDEX_PATTERN = "<.opensearch-sap-alerts-history-default-{now/d}-1>"; public static final String OPENSEARCH_DEFAULT_FINDINGS_INDEX = ".opensearch-sap-findings-default"; + public static final String OPENSEARCH_DEFAULT_ALL_FINDINGS_INDICES_PATTERN = ".opensearch-sap-findings-default*"; public static final String OPENSEARCH_DEFAULT_FINDINGS_INDEX_PATTERN = "<.opensearch-sap-findings-default-{now/d}-1>"; private static Map detectorTypeToIndicesMapping; @@ -41,10 +43,16 @@ public class DetectorMonitorConfig { Locale.getDefault(), ".opensearch-sap-%s-alerts*", detectorType.getDetectorType()); String findingsIndex = String.format( Locale.getDefault(), ".opensearch-sap-%s-findings", detectorType.getDetectorType()); + String allFindingsIndicesPattern = String.format( + Locale.getDefault(), ".opensearch-sap-%s-findings*", detectorType.getDetectorType()); String findingsIndexPattern = String.format( Locale.getDefault(), "<.opensearch-sap-%s-findings-{now/d}-1>", detectorType.getDetectorType()); - MonitorConfig monitor = new MonitorConfig(alertsIndex, alertsHistoryIndex, alertsHistoryIndexPattern, allAlertsIndicesPattern, findingsIndex, findingsIndexPattern, ruleIndex); + MonitorConfig monitor = new MonitorConfig( + alertsIndex, alertsHistoryIndex, alertsHistoryIndexPattern, allAlertsIndicesPattern, + findingsIndex, findingsIndexPattern, allFindingsIndicesPattern, + ruleIndex + ); detectorTypeToIndicesMapping.put(detectorType.getDetectorType(), monitor); }); } @@ -76,7 +84,14 @@ public static String getAlertsHistoryIndexPattern(String detectorType) { public static String getAllAlertsIndicesPattern(String detectorType) { return detectorTypeToIndicesMapping.containsKey(detectorType) ? detectorTypeToIndicesMapping.get(detectorType).getAllAlertsIndicesPattern() : - "*"; + OPENSEARCH_DEFAULT_ALL_ALERT_INDICES_PATTERN; + } + + public static List getAllAlertsIndicesPatternForAllTypes() { + return detectorTypeToIndicesMapping.entrySet() + .stream() + .map(e -> e.getValue().getAllAlertsIndicesPattern()) + .collect(Collectors.toList()); } public static String getFindingsIndex(String detectorType) { @@ -85,6 +100,19 @@ public static String getFindingsIndex(String detectorType) { OPENSEARCH_DEFAULT_FINDINGS_INDEX; } + public static String getAllFindingsIndicesPattern(String detectorType) { + return detectorTypeToIndicesMapping.containsKey(detectorType) ? + detectorTypeToIndicesMapping.get(detectorType).getAllFindingsIndicesPattern() : + OPENSEARCH_DEFAULT_ALL_FINDINGS_INDICES_PATTERN; + } + + public static List getAllFindingsIndicesPatternForAllTypes() { + return detectorTypeToIndicesMapping.entrySet() + .stream() + .map(e -> e.getValue().getAllFindingsIndicesPattern()) + .collect(Collectors.toList()); + } + public static String getFindingsIndexPattern(String detectorType) { return detectorTypeToIndicesMapping.containsKey(detectorType) ? detectorTypeToIndicesMapping.get(detectorType).getFindingsIndexPattern() : @@ -106,6 +134,7 @@ public static class MonitorConfig { private final String allAlertsIndicesPattern; private final String findingIndex; private final String findingsIndexPattern; + private final String allFindingsIndicesPattern; private final String ruleIndex; private MonitorConfig( @@ -115,6 +144,7 @@ private MonitorConfig( String allAlertsIndicesPattern, String findingsIndex, String findingsIndexPattern, + String allFindingsIndicesPattern, String ruleIndex ) { this.alertsIndex = alertsIndex; @@ -123,6 +153,7 @@ private MonitorConfig( this.allAlertsIndicesPattern = allAlertsIndicesPattern; this.findingIndex = findingsIndex; this.findingsIndexPattern = findingsIndexPattern; + this.allFindingsIndicesPattern = allFindingsIndicesPattern; this.ruleIndex = ruleIndex; } @@ -150,6 +181,10 @@ public String getFindingsIndexPattern() { return findingsIndexPattern; } + public String getAllFindingsIndicesPattern() { + return allFindingsIndicesPattern; + } + public String getRuleIndex() { return ruleIndex; } diff --git a/src/main/java/org/opensearch/securityanalytics/findings/FindingsService.java b/src/main/java/org/opensearch/securityanalytics/findings/FindingsService.java index 7960c514d..a696971da 100644 --- a/src/main/java/org/opensearch/securityanalytics/findings/FindingsService.java +++ b/src/main/java/org/opensearch/securityanalytics/findings/FindingsService.java @@ -91,7 +91,7 @@ public void onFailure(Exception e) { FindingsService.this.getFindingsByMonitorIds( monitorToDetectorMapping, monitorIds, - DetectorMonitorConfig.getFindingsIndex(detector.getDetectorType()), + DetectorMonitorConfig.getAllFindingsIndicesPattern(detector.getDetectorType()), table, getFindingsResponseListener ); @@ -183,7 +183,7 @@ public void getFindings( FindingsService.this.getFindingsByMonitorIds( monitorToDetectorMapping, allMonitorIds, - DetectorMonitorConfig.getFindingsIndex(detectorType.getDetectorType()), + DetectorMonitorConfig.getAllFindingsIndicesPattern(detectorType.getDetectorType()), table, new ActionListener<>() { @Override diff --git a/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java b/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java index ad6bbd240..98fdc08e9 100644 --- a/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java +++ b/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java @@ -55,9 +55,6 @@ public class DetectorIndexManagementService extends AbstractLifecycleComponent i private Logger logger = LogManager.getLogger(DetectorIndexManagementService.class); - private static final String ALERT_HISTORY_ALL = ".opensearch-sap-alerts-history-*"; - private static final String FINDING_HISTORY_ALL = ".opensearch-sap-findings-*"; - private final Client client; private final ThreadPool threadPool; private final ClusterService clusterService; @@ -235,7 +232,7 @@ private String executorName() { return ThreadPool.Names.MANAGEMENT; } - private void deleteOldIndices(String tag, String indices) { + private void deleteOldIndices(String tag, String... indices) { logger.error("info deleteOldIndices"); ClusterStateRequest clusterStateRequest = new ClusterStateRequest() .clear() @@ -250,7 +247,7 @@ private void deleteOldIndices(String tag, String indices) { public void onResponse(ClusterStateResponse clusterStateResponse) { if (!clusterStateResponse.getState().metadata().getIndices().isEmpty()) { List indicesToDelete = getIndicesToDelete(clusterStateResponse); - logger.info("Deleting old " + tag + " indices viz $indicesToDelete"); + logger.info("Checking if we should delete " + tag + " indices: [" + indicesToDelete + "]"); deleteAllOldHistoryIndices(indicesToDelete); } else { logger.info("No Old " + tag + " Indices to delete"); @@ -269,12 +266,14 @@ private List getIndicesToDelete(ClusterStateResponse clusterStateRespons List indicesToDelete = new ArrayList<>(); for (ObjectCursor in : clusterStateResponse.getState().metadata().indices().values()) { IndexMetadata indexMetaData = in.value; - indicesToDelete.add( - getHistoryIndexToDelete(indexMetaData, alertHistoryRetentionPeriod.millis(), alertHistoryIndices, alertHistoryEnabled) - ); - indicesToDelete.add( - getHistoryIndexToDelete(indexMetaData, findingHistoryRetentionPeriod.millis(), findingHistoryIndices, findingHistoryEnabled) - ); + String indexToDelete = getHistoryIndexToDelete(indexMetaData, alertHistoryRetentionPeriod.millis(), alertHistoryIndices, alertHistoryEnabled); + if (indexToDelete != null) { + indicesToDelete.add(indexToDelete); + } + indexToDelete = getHistoryIndexToDelete(indexMetaData, findingHistoryRetentionPeriod.millis(), findingHistoryIndices, findingHistoryEnabled); + if (indexToDelete != null) { + indicesToDelete.add(indexToDelete); + } } return indicesToDelete; } @@ -319,15 +318,17 @@ private void deleteAllOldHistoryIndices(List indicesToDelete) { public void onResponse(AcknowledgedResponse deleteIndicesResponse) { if (!deleteIndicesResponse.isAcknowledged()) { logger.error( - "Could not delete one or more Alerting/Finding history indices: $indicesToDelete. Retrying one by one." + "Could not delete one or more Alerting/Finding history indices: [" + indicesToDelete + "]. Retrying one by one." ); deleteOldHistoryIndex(indicesToDelete); + } else { + logger.info("Succsessfuly deleted indices: [" + indicesToDelete + "]"); } } @Override public void onFailure(Exception e) { - logger.error("Delete for Alerting/Finding History Indices $indicesToDelete Failed. Retrying one By one."); + logger.error("Delete for Alerting/Finding History Indices failed: [" + indicesToDelete + "]. Retrying one By one."); deleteOldHistoryIndex(indicesToDelete); } } @@ -351,7 +352,7 @@ public void onResponse(AcknowledgedResponse acknowledgedResponse) { @Override public void onFailure(Exception e) { - logger.debug("Exception ${e.message} while deleting the index " + index); + logger.debug("Exception: [" + e.getMessage() + "] while deleting the index " + index); } } ); @@ -360,12 +361,12 @@ public void onFailure(Exception e) { private void rolloverAndDeleteAlertHistoryIndices() { if (alertHistoryEnabled) rolloverAlertHistoryIndices(); - deleteOldIndices("History", ALERT_HISTORY_ALL); + deleteOldIndices("Alert", DetectorMonitorConfig.getAllAlertsIndicesPatternForAllTypes().toArray(new String[0])); } private void rolloverAndDeleteFindingHistoryIndices() { if (findingHistoryEnabled) rolloverFindingHistoryIndices(); - deleteOldIndices("Finding", FINDING_HISTORY_ALL); + deleteOldIndices("Finding", DetectorMonitorConfig.getAllFindingsIndicesPatternForAllTypes().toArray(new String[0])); } private void rolloverIndex( @@ -393,13 +394,13 @@ private void rolloverIndex( @Override public void onResponse(RolloverResponse rolloverResponse) { if (!rolloverResponse.isRolledOver()) { - logger.info(index + "not rolled over. Conditions were: ${response.conditionStatus}"); + logger.info(index + "not rolled over. Conditions were: " + rolloverResponse.getConditionStatus()); } } @Override public void onFailure(Exception e) { - logger.error(index + " not roll over failed."); + logger.error("rollover failed for index [" + index + "]."); } } ); @@ -417,9 +418,9 @@ private void rolloverAlertHistoryIndices() { private void rolloverFindingHistoryIndices() { for (HistoryIndexInfo h : findingHistoryIndices) { rolloverIndex( - h.isInitialized, h.indexAlias, - h.indexPattern, h.indexMappings, - h.maxDocs, h.maxAge + h.isInitialized, h.indexAlias, + h.indexPattern, h.indexMappings, + h.maxDocs, h.maxAge ); } } diff --git a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java index 20111d6eb..051f7f26b 100644 --- a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java +++ b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java @@ -1120,7 +1120,7 @@ public List getAlertIndices(String detectorType) throws IOException { } public List getFindingIndices(String detectorType) throws IOException { - Response response = client().performRequest(new Request("GET", "/_cat/indices/" + DetectorMonitorConfig.getFindingsIndex(detectorType) + "?format=json")); + Response response = client().performRequest(new Request("GET", "/_cat/indices/" + DetectorMonitorConfig.getAllFindingsIndicesPattern(detectorType) + "?format=json")); XContentParser xcp = createParser(XContentType.JSON.xContent(), response.getEntity().getContent()); List responseList = xcp.list(); List indices = new ArrayList<>(); diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java index eb9ce5778..7a382662a 100644 --- a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java +++ b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java @@ -42,6 +42,7 @@ import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_INDEX_MAX_AGE; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_MAX_DOCS; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_RETENTION_PERIOD; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_ROLLOVER_PERIOD; public class AlertsIT extends SecurityAnalyticsRestTestCase { @@ -495,7 +496,74 @@ public void testAlertHistoryRollover_maxAge() throws IOException, InterruptedExc indexDoc(index, "1", randomDoc()); - client().performRequest(new Request("POST", "_refresh")); + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + + request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + hits = new ArrayList<>(); + + while (hits.size() == 0) { + hits = executeSearch(DetectorMonitorConfig.getAlertsIndex(randomDetectorType()), request); + } + + List alertIndices = getAlertIndices(detector.getDetectorType()); + while(alertIndices.size() < 3) { + alertIndices = getAlertIndices(detector.getDetectorType()); + Thread.sleep(1000); + } + assertTrue("Did not find 3 alert indices", alertIndices.size() >= 3); + + } + + public void testAlertHistoryRollover_maxAge_low_retention() throws IOException, InterruptedException { + updateClusterSetting(ALERT_HISTORY_ROLLOVER_PERIOD.getKey(), "1s"); + updateClusterSetting(ALERT_HISTORY_MAX_DOCS.getKey(), "1000"); + updateClusterSetting(ALERT_HISTORY_INDEX_MAX_AGE.getKey(), "1s"); + + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"windows\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + Detector detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of()))); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String createdId = responseBody.get("_id").toString(); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + createdId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + indexDoc(index, "1", randomDoc()); Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); Map executeResults = entityAsMap(executeResponse); @@ -521,6 +589,15 @@ public void testAlertHistoryRollover_maxAge() throws IOException, InterruptedExc } assertTrue("Did not find 3 alert indices", alertIndices.size() >= 3); + updateClusterSetting(ALERT_HISTORY_INDEX_MAX_AGE.getKey(), "1000s"); + updateClusterSetting(ALERT_HISTORY_RETENTION_PERIOD.getKey(), "1s"); + + while(alertIndices.size() != 1) { + alertIndices = getAlertIndices(detector.getDetectorType()); + Thread.sleep(1000); + } + + assertTrue("Did not find 3 alert indices", alertIndices.size() == 1); } public void testAlertHistoryRollover_maxDocs() throws IOException, InterruptedException { @@ -605,4 +682,98 @@ public void testAlertHistoryRollover_maxDocs() throws IOException, InterruptedEx } + public void testGetAlertsFromAllIndices() throws IOException, InterruptedException { + updateClusterSetting(ALERT_HISTORY_ROLLOVER_PERIOD.getKey(), "1s"); + updateClusterSetting(ALERT_HISTORY_MAX_DOCS.getKey(), "1"); + + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + randomDetectorType() + "\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + Detector detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of()))); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String detectorId = responseBody.get("_id").toString(); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + indexDoc(index, "1", randomDoc()); + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + + request = "{\n" + + " \"query\" : {\n" + + " \"match_all\":{\n" + + " }\n" + + " }\n" + + "}"; + hits = new ArrayList<>(); + + while (hits.size() == 0) { + hits = executeSearch(DetectorMonitorConfig.getAlertsIndex(randomDetectorType()), request); + } + + Map params = new HashMap<>(); + params.put("detector_id", detectorId); + Response getAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.ALERTS_BASE_URI, params, null); + Map getAlertsBody = asMap(getAlertsResponse); + // TODO enable asserts here when able + Assert.assertEquals(1, getAlertsBody.get("total_alerts")); + String alertId = (String) ((ArrayList>) getAlertsBody.get("alerts")).get(0).get("id"); + String _detectorId = (String) ((ArrayList>) getAlertsBody.get("alerts")).get(0).get("detector_id"); + + // Ack alert to move it to history index + acknowledgeAlert(alertId, detectorId); + + List alertIndices = getAlertIndices(detector.getDetectorType()); + // alertIndex + 2 alertHistory indices + while(alertIndices.size() < 3) { + alertIndices = getAlertIndices(detector.getDetectorType()); + Thread.sleep(1000); + } + assertTrue("Did not find 3 alert indices", alertIndices.size() >= 3); + + // Index another doc to generate new alert in alertIndex + indexDoc(index, "2", randomDoc()); + + executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + executeResults = entityAsMap(executeResponse); + noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + + client().performRequest(new Request("POST", DetectorMonitorConfig.getAlertsIndex(randomDetectorType()) + "/_refresh")); + + getAlertsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.ALERTS_BASE_URI, params, null); + getAlertsBody = asMap(getAlertsResponse); + // 1 from alertIndex and 1 from history index + Assert.assertEquals(2, getAlertsBody.get("total_alerts")); + } } \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java index dde7251ae..c85e79ef7 100644 --- a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java +++ b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java @@ -32,6 +32,7 @@ import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_INDEX_MAX_AGE; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_MAX_DOCS; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_RETENTION_PERIOD; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_ROLLOVER_PERIOD; public class FindingIT extends SecurityAnalyticsRestTestCase { @@ -302,9 +303,9 @@ public void testGetFindings_rolloverByMaxAge_success() throws IOException, Inter Map getFindingsBody = entityAsMap(getFindingsResponse); Assert.assertEquals(1, getFindingsBody.get("total_findings")); - List findingIndices = getAlertIndices(detector.getDetectorType()); + List findingIndices = getFindingIndices(detector.getDetectorType()); while(findingIndices.size() < 2) { - findingIndices = getAlertIndices(detector.getDetectorType()); + findingIndices = getFindingIndices(detector.getDetectorType()); Thread.sleep(1000); } assertTrue("Did not find 3 alert indices", findingIndices.size() >= 2); @@ -365,11 +366,96 @@ public void testGetFindings_rolloverByMaxDoc_success() throws IOException, Inter Map getFindingsBody = entityAsMap(getFindingsResponse); Assert.assertEquals(1, getFindingsBody.get("total_findings")); - List findingIndices = getAlertIndices(detector.getDetectorType()); + List findingIndices = getFindingIndices(detector.getDetectorType()); while(findingIndices.size() < 2) { - findingIndices = getAlertIndices(detector.getDetectorType()); + findingIndices = getFindingIndices(detector.getDetectorType()); Thread.sleep(1000); } assertTrue("Did not find 3 alert indices", findingIndices.size() >= 2); } + + public void testGetFindings_rolloverByMaxDoc_short_retention_success() throws IOException, InterruptedException { + updateClusterSetting(FINDING_HISTORY_ROLLOVER_PERIOD.getKey(), "1s"); + updateClusterSetting(FINDING_HISTORY_MAX_DOCS.getKey(), "1"); + + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"windows\", " + + " \"partial\":true" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + Detector detector = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of()))); + + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + String detectorId = responseBody.get("_id").toString(); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + SearchHit hit = hits.get(0); + + String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + indexDoc(index, "1", randomDoc()); + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + // Call GetFindings API + Map params = new HashMap<>(); + params.put("detector_id", detectorId); + client().performRequest(new Request("POST", "_refresh")); + Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + Map getFindingsBody = entityAsMap(getFindingsResponse); + Assert.assertEquals(1, getFindingsBody.get("total_findings")); + + List findingIndices = getFindingIndices(detector.getDetectorType()); + while(findingIndices.size() < 2) { + findingIndices = getFindingIndices(detector.getDetectorType()); + Thread.sleep(1000); + } + assertTrue("Did not find 3 findings indices", findingIndices.size() >= 2); + + updateClusterSetting(FINDING_HISTORY_RETENTION_PERIOD.getKey(), "1s"); + updateClusterSetting(FINDING_HISTORY_MAX_DOCS.getKey(), "1000"); + while(findingIndices.size() != 1) { + findingIndices = getFindingIndices(detector.getDetectorType()); + Thread.sleep(1000); + } + + assertTrue("Found finding indices but expected none", findingIndices.size() == 1); + + // Exec monitor again to make sure that current + indexDoc(index, "2", randomDoc()); + + executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + executeResults = entityAsMap(executeResponse); + + noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + client().performRequest(new Request("POST", "_refresh")); + getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); + getFindingsBody = entityAsMap(getFindingsResponse); + Assert.assertEquals(1, getFindingsBody.get("total_findings")); + } }