diff --git a/src/main/resources/mappings/finding_mapping.json b/src/main/resources/mappings/finding_mapping.json index c9386b2ef..421dc202c 100644 --- a/src/main/resources/mappings/finding_mapping.json +++ b/src/main/resources/mappings/finding_mapping.json @@ -1,7 +1,7 @@ { "dynamic": "strict", "_meta" : { - "schema_version": 1 + "schema_version": 2 }, "properties": { "schema_version": { @@ -51,6 +51,15 @@ }, "timestamp": { "type": "long" + }, + "correlated_doc_ids": { + "type" : "text", + "analyzer": "whitespace", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } } } } \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java index 1a4595125..db08dbfa5 100644 --- a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java +++ b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java @@ -80,6 +80,14 @@ import static org.opensearch.securityanalytics.TestHelpers.sumAggregationTestRule; import static org.opensearch.securityanalytics.TestHelpers.productIndexAvgAggRule; import static org.opensearch.securityanalytics.TestHelpers.windowsIndexMapping; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_INDEX_MAX_AGE; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_MAX_DOCS; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_RETENTION_PERIOD; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ALERT_HISTORY_ROLLOVER_PERIOD; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_INDEX_MAX_AGE; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_MAX_DOCS; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_RETENTION_PERIOD; +import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_ROLLOVER_PERIOD; import static org.opensearch.securityanalytics.util.RuleTopicIndices.ruleTopicIndexSettings; public class SecurityAnalyticsRestTestCase extends OpenSearchRestTestCase { @@ -1629,4 +1637,18 @@ protected void createSampleDatastream(String datastreamName, String mappings, bo createDatastreamAPI(datastreamName); } + + + protected void restoreAlertsFindingsIMSettings() throws IOException { + updateClusterSetting(ALERT_HISTORY_ROLLOVER_PERIOD.getKey(), "720m"); + updateClusterSetting(ALERT_HISTORY_MAX_DOCS.getKey(), "100000"); + updateClusterSetting(ALERT_HISTORY_INDEX_MAX_AGE.getKey(), "60d"); + updateClusterSetting(ALERT_HISTORY_RETENTION_PERIOD.getKey(), "60d"); + + updateClusterSetting(FINDING_HISTORY_ROLLOVER_PERIOD.getKey(), "720m"); + updateClusterSetting(FINDING_HISTORY_MAX_DOCS.getKey(), "100000"); + updateClusterSetting(FINDING_HISTORY_INDEX_MAX_AGE.getKey(), "60d"); + updateClusterSetting(FINDING_HISTORY_RETENTION_PERIOD.getKey(), "60d"); + + } } \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java index f3e0ff922..209374232 100644 --- a/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java +++ b/src/test/java/org/opensearch/securityanalytics/alerts/AlertsIT.java @@ -655,6 +655,7 @@ public void testAlertHistoryRollover_maxAge() throws IOException, InterruptedExc } assertTrue("Did not find 3 alert indices", alertIndices.size() >= 3); + restoreAlertsFindingsIMSettings(); } public void testAlertHistoryRollover_maxAge_low_retention() throws IOException, InterruptedException { @@ -733,6 +734,8 @@ public void testAlertHistoryRollover_maxAge_low_retention() throws IOException, } assertTrue("Did not find 3 alert indices", alertIndices.size() == 1); + + restoreAlertsFindingsIMSettings(); } public void testAlertHistoryRollover_maxDocs() throws IOException, InterruptedException { @@ -815,6 +818,7 @@ public void testAlertHistoryRollover_maxDocs() throws IOException, InterruptedEx } assertTrue("Did not find 3 alert indices", alertIndices.size() >= 3); + restoreAlertsFindingsIMSettings(); } public void testGetAlertsFromAllIndices() throws IOException, InterruptedException { @@ -910,5 +914,7 @@ public void testGetAlertsFromAllIndices() throws IOException, InterruptedExcepti getAlertsBody = asMap(getAlertsResponse); // 1 from alertIndex and 1 from history index Assert.assertEquals(2, getAlertsBody.get("total_alerts")); + + restoreAlertsFindingsIMSettings(); } } \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java index 0ec572b7a..ce22cdeff 100644 --- a/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java +++ b/src/test/java/org/opensearch/securityanalytics/findings/FindingIT.java @@ -305,9 +305,21 @@ public void testGetFindings_rolloverByMaxAge_success() throws IOException, Inter String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + // Execute monitor first time to create findings index/alias indexDoc(index, "1", randomDoc()); - Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + + // Wait for findings index to rollover first, to make sure that our rollover applied correct settings/mappings + List findingIndices = getFindingIndices(detector.getDetectorType()); + while(findingIndices.size() < 2) { + findingIndices = getFindingIndices(detector.getDetectorType()); + Thread.sleep(1000); + } + assertTrue("Did not find more then 2 finding indices", findingIndices.size() >= 2); + + // Execute monitor second time to insert finding in new rollover'd index + indexDoc(index, "2", randomDoc()); + executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); Map executeResults = entityAsMap(executeResponse); int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); @@ -317,14 +329,9 @@ public void testGetFindings_rolloverByMaxAge_success() throws IOException, Inter params.put("detector_id", detectorId); Response getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); Map getFindingsBody = entityAsMap(getFindingsResponse); - Assert.assertEquals(1, getFindingsBody.get("total_findings")); + Assert.assertEquals(2, getFindingsBody.get("total_findings")); - List findingIndices = getFindingIndices(detector.getDetectorType()); - while(findingIndices.size() < 2) { - findingIndices = getFindingIndices(detector.getDetectorType()); - Thread.sleep(1000); - } - assertTrue("Did not find 3 alert indices", findingIndices.size() >= 2); + restoreAlertsFindingsIMSettings(); } public void testGetFindings_rolloverByMaxDoc_success() throws IOException, InterruptedException { @@ -388,6 +395,8 @@ public void testGetFindings_rolloverByMaxDoc_success() throws IOException, Inter Thread.sleep(1000); } assertTrue("Did not find 3 alert indices", findingIndices.size() >= 2); + + restoreAlertsFindingsIMSettings(); } public void testGetFindings_rolloverByMaxDoc_short_retention_success() throws IOException, InterruptedException { @@ -473,5 +482,7 @@ public void testGetFindings_rolloverByMaxDoc_short_retention_success() throws IO getFindingsResponse = makeRequest(client(), "GET", SecurityAnalyticsPlugin.FINDINGS_BASE_URI + "/_search", params, null); getFindingsBody = entityAsMap(getFindingsResponse); Assert.assertEquals(1, getFindingsBody.get("total_findings")); + + restoreAlertsFindingsIMSettings(); } }