-
Notifications
You must be signed in to change notification settings - Fork 76
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
separate doc-level monitor query indices created by detectors #1324
Changes from 3 commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -5,6 +5,8 @@ | |
package org.opensearch.securityanalytics.config.monitors; | ||
|
||
import java.util.List; | ||
import java.util.Random; | ||
import java.util.UUID; | ||
import java.util.stream.Collectors; | ||
import org.opensearch.common.inject.Inject; | ||
import org.opensearch.securityanalytics.logtype.LogTypeService; | ||
|
@@ -22,7 +24,11 @@ public class DetectorMonitorConfig { | |
public static final String OPENSEARCH_SAP_RULE_INDEX_TEMPLATE = ".opensearch-sap-detectors-queries-index-template"; | ||
|
||
public static String getRuleIndex(String logType) { | ||
return String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries", logType); | ||
return String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries*", logType); | ||
} | ||
|
||
public static String getRuleIndexOptimized(String logType) { | ||
return String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries-optimized-%s", logType, UUID.randomUUID()); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. we are moving from index patttern to index? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. no. this method is utilized by few integ tests & hence the change. |
||
} | ||
|
||
public static String getAlertsIndex(String logType) { | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -156,6 +156,8 @@ public class TransportIndexDetectorAction extends HandledTransportAction<IndexDe | |
|
||
private volatile Boolean enabledWorkflowUsage; | ||
|
||
private volatile Boolean enableDetectorWithDedicatedQueryIndices; | ||
|
||
private final Settings settings; | ||
|
||
private final NamedWriteableRegistry namedWriteableRegistry; | ||
|
@@ -202,11 +204,13 @@ public TransportIndexDetectorAction(TransportService transportService, | |
this.indexTimeout = SecurityAnalyticsSettings.INDEX_TIMEOUT.get(this.settings); | ||
this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); | ||
this.enabledWorkflowUsage = SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE.get(this.settings); | ||
this.enableDetectorWithDedicatedQueryIndices = SecurityAnalyticsSettings.ENABLE_DETECTORS_WITH_DEDICATED_QUERY_INDICES.get(this.settings); | ||
this.monitorService = new MonitorService(client); | ||
this.workflowService = new WorkflowService(client, monitorService); | ||
|
||
this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES, this::setFilterByEnabled); | ||
this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE, this::setEnabledWorkflowUsage); | ||
this.clusterService.getClusterSettings().addSettingsUpdateConsumer(SecurityAnalyticsSettings.ENABLE_DETECTORS_WITH_DEDICATED_QUERY_INDICES, this::setEnabledDetectorsWithDedicatedQueryIndices); | ||
this.exceptionChecker = exceptionChecker; | ||
} | ||
|
||
|
@@ -793,7 +797,7 @@ private IndexMonitorRequest createDocLevelMonitorRequest(List<Pair<String, Rule> | |
detector.getAlertsHistoryIndex(), | ||
detector.getAlertsHistoryIndexPattern(), | ||
DetectorMonitorConfig.getRuleIndexMappingsByType(), | ||
true), PLUGIN_OWNER_FIELD); | ||
true), enableDetectorWithDedicatedQueryIndices, PLUGIN_OWNER_FIELD); | ||
|
||
return new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null); | ||
} | ||
|
@@ -887,14 +891,14 @@ private IndexMonitorRequest createDocLevelMonitorMatchAllRequest( | |
|
||
Monitor monitor = new Monitor(monitorId, Monitor.NO_VERSION, monitorName, false, detector.getSchedule(), detector.getLastUpdateTime(), null, | ||
Monitor.MonitorType.DOC_LEVEL_MONITOR.getValue(), detector.getUser(), 1, docLevelMonitorInputs, triggers, Map.of(), | ||
new DataSources(detector.getRuleIndex(), | ||
new DataSources(enableDetectorWithDedicatedQueryIndices? detector.getRuleIndex() + "_chained_findings": detector.getRuleIndex(), | ||
detector.getFindingsIndex(), | ||
detector.getFindingsIndexPattern(), | ||
detector.getAlertsIndex(), | ||
detector.getAlertsHistoryIndex(), | ||
detector.getAlertsHistoryIndexPattern(), | ||
DetectorMonitorConfig.getRuleIndexMappingsByType(), | ||
true), PLUGIN_OWNER_FIELD); | ||
true), enableDetectorWithDedicatedQueryIndices, PLUGIN_OWNER_FIELD); | ||
|
||
return new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null); | ||
} | ||
|
@@ -1068,7 +1072,7 @@ public void onResponse(GetIndexMappingsResponse getIndexMappingsResponse) { | |
detector.getAlertsHistoryIndex(), | ||
detector.getAlertsHistoryIndexPattern(), | ||
DetectorMonitorConfig.getRuleIndexMappingsByType(), | ||
true), PLUGIN_OWNER_FIELD); | ||
true), false, PLUGIN_OWNER_FIELD); | ||
|
||
listener.onResponse(new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null)); | ||
} | ||
|
@@ -1252,7 +1256,13 @@ void createDetector() { | |
request.getDetector().setAlertsHistoryIndexPattern(DetectorMonitorConfig.getAlertsHistoryIndexPattern(ruleTopic)); | ||
request.getDetector().setFindingsIndex(DetectorMonitorConfig.getFindingsIndex(ruleTopic)); | ||
request.getDetector().setFindingsIndexPattern(DetectorMonitorConfig.getFindingsIndexPattern(ruleTopic)); | ||
request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndex(ruleTopic)); | ||
|
||
if (enableDetectorWithDedicatedQueryIndices) { | ||
request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndexOptimized(ruleTopic)); | ||
} else { | ||
String ruleTopicIndex = DetectorMonitorConfig.getRuleIndex(ruleTopic); | ||
request.getDetector().setRuleIndex(ruleTopicIndex.substring(0, ruleTopicIndex.length()-1)); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. why did we change the existing code from There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
} | ||
|
||
User originalContextUser = this.user; | ||
log.debug("user from original context is {}", originalContextUser); | ||
|
@@ -1369,7 +1379,16 @@ void onGetResponse(Detector currentDetector, User user) { | |
request.getDetector().setAlertsHistoryIndexPattern(DetectorMonitorConfig.getAlertsHistoryIndexPattern(ruleTopic)); | ||
request.getDetector().setFindingsIndex(DetectorMonitorConfig.getFindingsIndex(ruleTopic)); | ||
request.getDetector().setFindingsIndexPattern(DetectorMonitorConfig.getFindingsIndexPattern(ruleTopic)); | ||
request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndex(ruleTopic)); | ||
if (currentDetector.getRuleIndex().contains("optimized")) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. if name contains is this if else condition flipped? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. the |
||
request.getDetector().setRuleIndex(currentDetector.getRuleIndex()); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. plz add comment that if we turn OFF setting after turning on, updating detector will not change from optimized to normal and we need to re-create detector There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. added a comment. added a test too. |
||
} else { | ||
if (enableDetectorWithDedicatedQueryIndices) { | ||
request.getDetector().setRuleIndex(DetectorMonitorConfig.getRuleIndexOptimized(ruleTopic)); | ||
} else { | ||
String ruleTopicIndex = DetectorMonitorConfig.getRuleIndex(ruleTopic); | ||
request.getDetector().setRuleIndex(ruleTopicIndex.substring(0, ruleTopicIndex.length() - 1)); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this line is very confusing There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. fixed it. |
||
} | ||
} | ||
request.getDetector().setUser(user); | ||
|
||
if (!detector.getInputs().isEmpty()) { | ||
|
@@ -1805,4 +1824,8 @@ private void setFilterByEnabled(boolean filterByEnabled) { | |
private void setEnabledWorkflowUsage(boolean enabledWorkflowUsage) { | ||
this.enabledWorkflowUsage = enabledWorkflowUsage; | ||
} | ||
|
||
private void setEnabledDetectorsWithDedicatedQueryIndices(boolean enabledDetectorsWithDedicatedQueryIndices) { | ||
this.enableDetectorWithDedicatedQueryIndices = enabledDetectorsWithDedicatedQueryIndices; | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -69,22 +69,11 @@ public void testCreateDetectorWithThreatIntelEnabled_updateDetectorWithThreatInt | |
Detector detector = randomDetectorWithInputsAndThreatIntelAndTriggers(List.of(input), true, List.of(trigger)); | ||
Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); | ||
|
||
String request = "{\n" + | ||
" \"query\" : {\n" + | ||
" \"match_all\":{\n" + | ||
" }\n" + | ||
" }\n" + | ||
"}"; | ||
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. why has this been removed ? Plz dont reduce test coverage.. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. query indices are created & deleted now. So, it is difficult to assert number of queries in query index now. |
||
|
||
|
||
assertEquals(2, response.getHits().getTotalHits().value); | ||
|
||
assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); | ||
Map<String, Object> responseBody = asMap(createResponse); | ||
|
||
String detectorId = responseBody.get("_id").toString(); | ||
request = "{\n" + | ||
String request = "{\n" + | ||
" \"query\" : {\n" + | ||
" \"match\":{\n" + | ||
" \"_id\": \"" + detectorId + "\"\n" + | ||
|
@@ -269,22 +258,11 @@ public void testCreateDetectorWithThreatIntelDisabled_updateDetectorWithThreatIn | |
Detector detector = randomDetectorWithInputsAndThreatIntel(List.of(input), false); | ||
Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); | ||
|
||
String request = "{\n" + | ||
" \"query\" : {\n" + | ||
" \"match_all\":{\n" + | ||
" }\n" + | ||
" }\n" + | ||
"}"; | ||
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); | ||
|
||
|
||
assertEquals(1, response.getHits().getTotalHits().value); | ||
|
||
assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); | ||
Map<String, Object> responseBody = asMap(createResponse); | ||
|
||
String detectorId = responseBody.get("_id").toString(); | ||
request = "{\n" + | ||
String request = "{\n" + | ||
" \"query\" : {\n" + | ||
" \"match\":{\n" + | ||
" \"_id\": \"" + detectorId + "\"\n" + | ||
|
@@ -366,22 +344,11 @@ public void testCreateDetectorWithThreatIntelEnabledAndNoRules_triggerDetectionT | |
Detector detector = randomDetectorWithInputsAndThreatIntelAndTriggers(List.of(input), true, List.of(trigger)); | ||
Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); | ||
|
||
String request = "{\n" + | ||
" \"query\" : {\n" + | ||
" \"match_all\":{\n" + | ||
" }\n" + | ||
" }\n" + | ||
"}"; | ||
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); | ||
|
||
|
||
assertEquals(1, response.getHits().getTotalHits().value); | ||
|
||
assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); | ||
Map<String, Object> responseBody = asMap(createResponse); | ||
|
||
String detectorId = responseBody.get("_id").toString(); | ||
request = "{\n" + | ||
String request = "{\n" + | ||
" \"query\" : {\n" + | ||
" \"match\":{\n" + | ||
" \"_id\": \"" + detectorId + "\"\n" + | ||
|
@@ -460,22 +427,11 @@ public void testCreateDetectorWithThreatIntelEnabled_triggerDetectionTypeOnlyThr | |
Detector detector = randomDetectorWithInputsAndThreatIntelAndTriggers(List.of(input), true, List.of(trigger)); | ||
Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); | ||
|
||
String request = "{\n" + | ||
" \"query\" : {\n" + | ||
" \"match_all\":{\n" + | ||
" }\n" + | ||
" }\n" + | ||
"}"; | ||
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); | ||
|
||
|
||
assertEquals(1, response.getHits().getTotalHits().value); | ||
|
||
assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); | ||
Map<String, Object> responseBody = asMap(createResponse); | ||
|
||
String detectorId = responseBody.get("_id").toString(); | ||
request = "{\n" + | ||
String request = "{\n" + | ||
" \"query\" : {\n" + | ||
" \"match\":{\n" + | ||
" \"_id\": \"" + detectorId + "\"\n" + | ||
|
@@ -555,22 +511,11 @@ public void testCreateDetectorWithThreatIntelEnabled_triggerWithBothDetectionTyp | |
Detector detector = randomDetectorWithInputsAndThreatIntelAndTriggers(List.of(input), true, List.of(trigger)); | ||
Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); | ||
|
||
String request = "{\n" + | ||
" \"query\" : {\n" + | ||
" \"match_all\":{\n" + | ||
" }\n" + | ||
" }\n" + | ||
"}"; | ||
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); | ||
|
||
|
||
assertEquals(1, response.getHits().getTotalHits().value); | ||
|
||
assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); | ||
Map<String, Object> responseBody = asMap(createResponse); | ||
|
||
String detectorId = responseBody.get("_id").toString(); | ||
request = "{\n" + | ||
String request = "{\n" + | ||
" \"query\" : {\n" + | ||
" \"match\":{\n" + | ||
" \"_id\": \"" + detectorId + "\"\n" + | ||
|
@@ -647,22 +592,11 @@ public void testCreateDetectorWithThreatIntelDisabled_triggerWithThreatIntelDete | |
Detector detector = randomDetectorWithInputsAndThreatIntelAndTriggers(List.of(input), false, List.of(trigger)); | ||
Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); | ||
|
||
String request = "{\n" + | ||
" \"query\" : {\n" + | ||
" \"match_all\":{\n" + | ||
" }\n" + | ||
" }\n" + | ||
"}"; | ||
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); | ||
|
||
|
||
assertEquals(1, response.getHits().getTotalHits().value); | ||
|
||
assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); | ||
Map<String, Object> responseBody = asMap(createResponse); | ||
|
||
String detectorId = responseBody.get("_id").toString(); | ||
request = "{\n" + | ||
String request = "{\n" + | ||
" \"query\" : {\n" + | ||
" \"match\":{\n" + | ||
" \"_id\": \"" + detectorId + "\"\n" + | ||
|
@@ -739,22 +673,11 @@ public void testCreateDetectorWithThreatIntelDisabled_triggerWithRulesDetectionT | |
Detector detector = randomDetectorWithInputsAndThreatIntelAndTriggers(List.of(input), false, List.of(trigger)); | ||
Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); | ||
|
||
String request = "{\n" + | ||
" \"query\" : {\n" + | ||
" \"match_all\":{\n" + | ||
" }\n" + | ||
" }\n" + | ||
"}"; | ||
SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); | ||
|
||
|
||
assertEquals(1, response.getHits().getTotalHits().value); | ||
|
||
assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); | ||
Map<String, Object> responseBody = asMap(createResponse); | ||
|
||
String detectorId = responseBody.get("_id").toString(); | ||
request = "{\n" + | ||
String request = "{\n" + | ||
" \"query\" : {\n" + | ||
" \"match\":{\n" + | ||
" \"_id\": \"" + detectorId + "\"\n" + | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -266,6 +266,7 @@ void setDebugLogLevel() throws IOException, InterruptedException { | |
|
||
|
||
makeRequest(client(), "PUT", "_cluster/settings", Collections.emptyMap(), se, new BasicHeader("Content-Type", "application/json")); | ||
updateClusterSetting("plugins.security_analytics.enable_detectors_with_dedicated_query_indices", "true"); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. are there tests with setting turned off There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. added a test for setting turned off. |
||
} | ||
|
||
protected final List<String> clusterPermissions = List.of( | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
why
*
this is an alias
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
fixed it.