diff --git a/build.gradle b/build.gradle index a158763bb..48984a3f9 100644 --- a/build.gradle +++ b/build.gradle @@ -6,7 +6,7 @@ import org.opensearch.gradle.test.RestIntegTestTask buildscript { ext { - opensearch_version = System.getProperty("opensearch.version", "2.4.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "2.5.0-SNAPSHOT") isSnapshot = "true" == System.getProperty("build.snapshot", "true") buildVersionQualifier = System.getProperty("build.version_qualifier", "") version_tokens = opensearch_version.tokenize('-') diff --git a/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java b/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java index 02258c2aa..fc79c6454 100644 --- a/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java +++ b/src/main/java/org/opensearch/securityanalytics/config/monitors/DetectorMonitorConfig.java @@ -25,6 +25,8 @@ public class DetectorMonitorConfig { public static final String OPENSEARCH_DEFAULT_ALL_FINDINGS_INDICES_PATTERN = ".opensearch-sap-findings-default*"; public static final String OPENSEARCH_DEFAULT_FINDINGS_INDEX_PATTERN = "<.opensearch-sap-findings-default-{now/d}-1>"; + public static final String OPENSEARCH_SAP_RULE_INDEX_TEMPLATE = ".opensearch-sap-detectors-queries-index-template"; + private static Map detectorTypeToIndicesMapping; static { @@ -113,6 +115,13 @@ public static List getAllFindingsIndicesPatternForAllTypes() { .collect(Collectors.toList()); } + public static List getAllRuleIndices() { + return detectorTypeToIndicesMapping.entrySet() + .stream() + .map(e -> e.getValue().getRuleIndex()) + .collect(Collectors.toList()); + } + public static String getFindingsIndexPattern(String detectorType) { return detectorTypeToIndicesMapping.containsKey(detectorType.toLowerCase(Locale.ROOT)) ? detectorTypeToIndicesMapping.get(detectorType.toLowerCase(Locale.ROOT)).getFindingsIndexPattern() : @@ -145,8 +154,7 @@ private MonitorConfig( String findingsIndex, String findingsIndexPattern, String allFindingsIndicesPattern, - String ruleIndex - ) { + String ruleIndex) { this.alertsIndex = alertsIndex; this.alertsHistoryIndex = alertsHistoryIndex; this.alertsHistoryIndexPattern = alertsHistoryIndexPattern; diff --git a/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java b/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java index b356e5b54..52ce060d2 100644 --- a/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java +++ b/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java @@ -5,16 +5,17 @@ package org.opensearch.securityanalytics.mapper; -import java.util.Locale; +import java.util.Collection; +import java.util.Optional; import org.apache.commons.lang3.tuple.Pair; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.OpenSearchStatusException; -import org.opensearch.ResourceNotFoundException; import org.opensearch.action.ActionListener; import org.opensearch.action.admin.indices.mapping.get.GetMappingsRequest; import org.opensearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.opensearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.opensearch.action.support.GroupedActionListener; import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.IndicesAdminClient; import org.opensearch.cluster.metadata.MappingMetadata; @@ -74,10 +75,51 @@ public void onFailure(Exception e) { private void createMappingActionContinuation(ImmutableOpenMap indexMappings, String ruleTopic, String aliasMappings, boolean partial, ActionListener actionListener) { + int numOfIndices = indexMappings.size(); + + GroupedActionListener doCreateMappingActionsListener = new GroupedActionListener(new ActionListener>() { + @Override + public void onResponse(Collection response) { + // We will return ack==false if one of the requests returned that + // else return ack==true + Optional notAckd = response.stream().filter(e -> e.isAcknowledged() == false).findFirst(); + AcknowledgedResponse ack = new AcknowledgedResponse( + notAckd.isPresent() ? false : true + ); + actionListener.onResponse(ack); + } + + @Override + public void onFailure(Exception e) { + actionListener.onFailure( + new SecurityAnalyticsException( + "Failed applying mappings to index", RestStatus.INTERNAL_SERVER_ERROR, e) + ); + } + }, numOfIndices); + + indexMappings.forEach(iter -> { + String indexName = iter.key; + MappingMetadata mappingMetadata = iter.value; + // Try to apply mapping to index + doCreateMapping(indexName, mappingMetadata, ruleTopic, aliasMappings, partial, doCreateMappingActionsListener); + }); + } + + /** + * Applies alias mappings to index. + * @param indexName Index name + * @param mappingMetadata Index mappings + * @param ruleTopic Rule topic spcifying specific alias templates + * @param aliasMappings User-supplied alias mappings + * @param partial Partial flag indicating if we should apply mappings partially, in case source index doesn't have all paths specified in alias mappings + * @param actionListener actionListener used to return response/error + */ + private void doCreateMapping(String indexName, MappingMetadata mappingMetadata, String ruleTopic, String aliasMappings, boolean partial, ActionListener actionListener) { + PutMappingRequest request; try { - String indexName = indexMappings.iterator().next().key; String aliasMappingsJSON; // aliasMappings parameter has higher priority then ruleTopic if (aliasMappings != null) { @@ -86,7 +128,7 @@ private void createMappingActionContinuation(ImmutableOpenMap missingPathsInIndex = MapperUtils.validateIndexMappings(indexMappings, aliasMappingsJSON); + List missingPathsInIndex = MapperUtils.validateIndexMappings(indexName, mappingMetadata, aliasMappingsJSON); if(missingPathsInIndex.size() > 0) { // If user didn't allow partial apply, we should error out here diff --git a/src/main/java/org/opensearch/securityanalytics/mapper/MapperUtils.java b/src/main/java/org/opensearch/securityanalytics/mapper/MapperUtils.java index 2b3389928..90282d0b2 100644 --- a/src/main/java/org/opensearch/securityanalytics/mapper/MapperUtils.java +++ b/src/main/java/org/opensearch/securityanalytics/mapper/MapperUtils.java @@ -6,6 +6,7 @@ package org.opensearch.securityanalytics.mapper; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import org.apache.commons.lang3.tuple.Pair; import org.opensearch.cluster.metadata.MappingMetadata; @@ -89,24 +90,22 @@ public void onError(String error) { *
  • Alias mappings have to have property type=alias and path property has to exist *
  • Paths from alias mappings should exists in index mappings * - * @param indexMappings Index Mappings to which alias mappings will be applied - * @param aliasMappingsJSON Alias Mappings as JSON string + * @param indexName Source index name + * @param mappingMetadata Source index mapping to which alias mappings will be applied + * @param aliasMappingsJSON Alias mappings as JSON string * @return list of alias mappings paths which are missing in index mappings * */ - public static List validateIndexMappings(ImmutableOpenMap indexMappings, String aliasMappingsJSON) throws IOException { + public static List validateIndexMappings(String indexName, MappingMetadata mappingMetadata, String aliasMappingsJSON) throws IOException { // Check if index's mapping is empty - if (isIndexMappingsEmpty(indexMappings)) { - throw new IllegalArgumentException("Index mappings are empty"); + if (isIndexMappingsEmpty(mappingMetadata)) { + throw new IllegalArgumentException(String.format(Locale.ROOT, "Mappings for index [%s] are empty", indexName)); } // Get all paths (field names) to which we're going to apply aliases List paths = getAllPathsFromAliasMappings(aliasMappingsJSON); // Traverse Index Mappings and extract all fields(paths) - String indexName = indexMappings.iterator().next().key; - MappingMetadata mappingMetadata = indexMappings.get(indexName); - List flatFields = getAllNonAliasFieldsFromIndex(mappingMetadata); // Return list of paths from Alias Mappings which are missing in Index Mappings return paths.stream() @@ -164,11 +163,8 @@ public static List getAllNonAliasFieldsFromIndex(MappingMetadata mapping return mappingsTraverser.extractFlatNonAliasFields(); } - public static boolean isIndexMappingsEmpty(ImmutableOpenMap indexMappings) { - if (indexMappings.iterator().hasNext()) { - return indexMappings.iterator().next().value.getSourceAsMap().size() == 0; - } - throw new IllegalArgumentException("Invalid Index Mappings"); + public static boolean isIndexMappingsEmpty(MappingMetadata mappingMetadata) { + return mappingMetadata.getSourceAsMap().size() == 0; } public static Map getAliasMappingsWithFilter( diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportCreateIndexMappingsAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportCreateIndexMappingsAction.java index bef747909..162c45c16 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportCreateIndexMappingsAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportCreateIndexMappingsAction.java @@ -43,11 +43,6 @@ public TransportCreateIndexMappingsAction( protected void doExecute(Task task, CreateIndexMappingsRequest request, ActionListener actionListener) { this.threadPool.getThreadContext().stashContext(); - IndexMetadata index = clusterService.state().metadata().index(request.getIndexName()); - if (index == null) { - actionListener.onFailure(new IllegalStateException("Could not find index [" + request.getIndexName() + "]")); - return; - } mapperService.createMappingAction( request.getIndexName(), request.getRuleTopic(), diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportDeleteDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportDeleteDetectorAction.java index fe9dcb051..91f4a025f 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportDeleteDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportDeleteDetectorAction.java @@ -152,48 +152,7 @@ public void onResponse(Collection responses) { }).count() > 0) { onFailures(new OpenSearchStatusException("Monitor associated with detected could not be deleted", errorStatusSupplier.get())); } - ruleTopicIndices.countQueries(ruleIndex, new ActionListener<>() { - @Override - public void onResponse(SearchResponse response) { - if (response.isTimedOut()) { - log.info("Count response timed out"); - deleteDetectorFromConfig(detector.getId(), request.getRefreshPolicy()); - } else { - long count = response.getHits().getTotalHits().value; - - if (count == 0) { - try { - ruleTopicIndices.deleteRuleTopicIndex(ruleIndex, - new ActionListener<>() { - @Override - public void onResponse(AcknowledgedResponse response) { - deleteDetectorFromConfig(detector.getId(), request.getRefreshPolicy()); - } - - @Override - public void onFailure(Exception e) { - // error is suppressed as it is not a critical deletion - log.info(e.getMessage()); - deleteDetectorFromConfig(detector.getId(), request.getRefreshPolicy()); - } - }); - } catch (IOException e) { - deleteDetectorFromConfig(detector.getId(), request.getRefreshPolicy()); - } - } else { - deleteDetectorFromConfig(detector.getId(), request.getRefreshPolicy()); - } - } - } - - @Override - public void onFailure(Exception e) { - // error is suppressed as it is not a critical deletion - log.info(e.getMessage()); - - - } - }); + deleteDetectorFromConfig(detector.getId(), request.getRefreshPolicy()); } @Override diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java index 9c6774ac5..7f45b7db1 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java @@ -696,9 +696,9 @@ void createDetector() { if (!detector.getInputs().isEmpty()) { try { - ruleTopicIndices.initRuleTopicIndex(detector.getRuleIndex(), new ActionListener<>() { + ruleTopicIndices.initRuleTopicIndexTemplate(new ActionListener<>() { @Override - public void onResponse(CreateIndexResponse createIndexResponse) { + public void onResponse(AcknowledgedResponse acknowledgedResponse) { initRuleIndexAndImportRules(request, new ActionListener<>() { @Override @@ -802,9 +802,9 @@ void onGetResponse(Detector currentDetector, User user) { if (!detector.getInputs().isEmpty()) { try { - ruleTopicIndices.initRuleTopicIndex(detector.getRuleIndex(), new ActionListener<>() { + ruleTopicIndices.initRuleTopicIndexTemplate(new ActionListener<>() { @Override - public void onResponse(CreateIndexResponse createIndexResponse) { + public void onResponse(AcknowledgedResponse acknowledgedResponse) { initRuleIndexAndImportRules(request, new ActionListener<>() { @Override public void onResponse(List monitorResponses) { diff --git a/src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java b/src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java index 06d6e1c46..82a6e707d 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java +++ b/src/main/java/org/opensearch/securityanalytics/util/RuleTopicIndices.java @@ -4,26 +4,22 @@ */ package org.opensearch.securityanalytics.util; +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.ActionListener; -import org.opensearch.action.admin.indices.create.CreateIndexRequest; -import org.opensearch.action.admin.indices.create.CreateIndexResponse; -import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; -import org.opensearch.action.search.SearchRequest; -import org.opensearch.action.search.SearchResponse; +import org.opensearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.opensearch.action.support.master.AcknowledgedResponse; -import org.opensearch.client.AdminClient; import org.opensearch.client.Client; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.search.builder.SearchSourceBuilder; - -import java.io.IOException; -import java.nio.charset.Charset; -import java.util.Objects; +import org.opensearch.securityanalytics.config.monitors.DetectorMonitorConfig; public class RuleTopicIndices { private static final Logger log = LogManager.getLogger(DetectorIndices.class); @@ -37,40 +33,30 @@ public RuleTopicIndices(Client client, ClusterService clusterService) { this.clusterService = clusterService; } - public static String ruleTopicIndexMappings() throws IOException { - return new String(Objects.requireNonNull(DetectorIndices.class.getClassLoader().getResourceAsStream("mappings/detector-queries.json")).readAllBytes(), Charset.defaultCharset()); - } - public static String ruleTopicIndexSettings() throws IOException { return new String(Objects.requireNonNull(DetectorIndices.class.getClassLoader().getResourceAsStream("mappings/detector-settings.json")).readAllBytes(), Charset.defaultCharset()); } - public void initRuleTopicIndex(String ruleTopicIndex, ActionListener actionListener) throws IOException { - if (!ruleTopicIndexExists(ruleTopicIndex)) { - CreateIndexRequest indexRequest = new CreateIndexRequest(ruleTopicIndex) - .mapping(ruleTopicIndexMappings()) + public void initRuleTopicIndexTemplate(ActionListener actionListener) throws IOException { + if (!ruleTopicIndexTemplateExists()) { + // Compose list of all patterns to cover all query indices + List indexPatterns = new ArrayList<>(); + for(String ruleIndex : DetectorMonitorConfig.getAllRuleIndices()) { + indexPatterns.add(ruleIndex + "*"); + } + PutIndexTemplateRequest indexRequest = + new PutIndexTemplateRequest(DetectorMonitorConfig.OPENSEARCH_SAP_RULE_INDEX_TEMPLATE) + .patterns(indexPatterns) .settings(Settings.builder().loadFromSource(ruleTopicIndexSettings(), XContentType.JSON).build()); - client.admin().indices().create(indexRequest, actionListener); + client.admin().indices().putTemplate(indexRequest, actionListener); } else { - actionListener.onResponse(new CreateIndexResponse(true, true, ruleTopicIndex)); + actionListener.onResponse(new AcknowledgedResponse(true)); } } - public void deleteRuleTopicIndex(String ruleTopicIndex, ActionListener actionListener) throws IOException { - if (ruleTopicIndexExists(ruleTopicIndex)) { - DeleteIndexRequest request = new DeleteIndexRequest(ruleTopicIndex); - client.admin().indices().delete(request, actionListener); - } - } - - public void countQueries(String ruleTopicIndex, ActionListener listener) { - SearchRequest request = new SearchRequest(ruleTopicIndex) - .source(new SearchSourceBuilder().size(0)); - client.search(request, listener); - } - - public boolean ruleTopicIndexExists(String ruleTopicIndex) { + public boolean ruleTopicIndexTemplateExists() { ClusterState clusterState = clusterService.state(); - return clusterState.getRoutingTable().hasIndex(ruleTopicIndex); + return clusterState.metadata().templates() + .get(DetectorMonitorConfig.OPENSEARCH_SAP_RULE_INDEX_TEMPLATE) != null; } } \ No newline at end of file diff --git a/src/main/resources/mappings/detector-queries.json b/src/main/resources/mappings/detector-queries.json deleted file mode 100644 index 7f0602df7..000000000 --- a/src/main/resources/mappings/detector-queries.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "_meta": { - "schema_version": 1 - }, - "properties": { - "query": { - "type": "percolator_ext" - }, - "monitor_id": { - "type": "text" - }, - "index": { - "type": "text" - } - } -} \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java index 44ed59c41..61192251c 100644 --- a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java +++ b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java @@ -108,6 +108,15 @@ protected void createRuleTopicIndex(String detectorType, String additionalMappin } } + protected String createDetector(Detector detector) throws IOException { + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + + return responseBody.get("_id").toString(); + } + protected final List clusterPermissions = List.of( "cluster:admin/opensearch/securityanalytics/detector/*", "cluster:admin/opendistro/alerting/alerts/*", @@ -151,6 +160,17 @@ protected String createTestIndex(RestClient client, String index, String mapping return index; } + protected String createDocumentWithNFields(int numOfFields) { + StringBuilder doc = new StringBuilder(); + doc.append("{"); + for(int i = 0; i < numOfFields - 1; i++) { + doc.append("\"id").append(i).append("\": 5,"); + } + doc.append("\"last_field\": 100 }"); + + return doc.toString(); + } + protected Response makeRequest(RestClient client, String method, String endpoint, Map params, HttpEntity entity, Header... headers) throws IOException { Request request = new Request(method, endpoint); @@ -1211,6 +1231,25 @@ public List getAlertIndices(String detectorType) throws IOException { return indices; } + public List getQueryIndices(String detectorType) throws IOException { + Response response = client().performRequest(new Request("GET", "/_cat/indices/" + DetectorMonitorConfig.getRuleIndex(detectorType) + "*?format=json")); + XContentParser xcp = createParser(XContentType.JSON.xContent(), response.getEntity().getContent()); + List responseList = xcp.list(); + List indices = new ArrayList<>(); + for (Object o : responseList) { + if (o instanceof Map) { + ((Map) o).forEach((BiConsumer) + (o1, o2) -> { + if (o1.equals("index")) { + indices.add((String) o2); + } + }); + } + } + return indices; + } + + public List getFindingIndices(String detectorType) throws IOException { Response response = client().performRequest(new Request("GET", "/_cat/indices/" + DetectorMonitorConfig.getAllFindingsIndicesPattern(detectorType) + "?format=json")); XContentParser xcp = createParser(XContentType.JSON.xContent(), response.getEntity().getContent()); @@ -1291,4 +1330,16 @@ protected void createNetflowLogIndex(String indexName) throws IOException { response = client().performRequest(new Request("POST", "_refresh")); assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); } + + + private Map getIndexAPI(String index) throws IOException { + Response resp = makeRequest(client(), "GET", "/" + index + "?expand_wildcards=all", Collections.emptyMap(), null); + return asMap(resp); + } + + private Map getIndexSettingsAPI(String index) throws IOException { + Response resp = makeRequest(client(), "GET", "/" + index + "/_settings?expand_wildcards=all", Collections.emptyMap(), null); + Map respMap = asMap(resp); + return respMap; + } } \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 1a07dbcda..d236f28a7 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -64,7 +64,11 @@ public static Detector randomDetectorWithTriggers(List rules, List rules, List triggers, List inputIndices) { + DetectorInput input = new DetectorInput("windows detector for security analytics", inputIndices, Collections.emptyList(), + rules.stream().map(DetectorRule::new).collect(Collectors.toList())); + return randomDetector(null, null, null, List.of(input), triggers, null, null, null, null); + } public static Detector randomDetectorWithInputsAndTriggers(List inputs, List triggers) { return randomDetector(null, null, null, inputs, triggers, null, null, null, null); } diff --git a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java index 7b454e77a..431b89564 100644 --- a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java @@ -241,7 +241,7 @@ public void testCreateIndexMappingsIndexMappingsEmpty() throws IOException { try { client().performRequest(request); } catch (ResponseException e) { - assertTrue(e.getMessage().contains("Index mappings are empty")); + assertTrue(e.getMessage().contains("Mappings for index [my_index_alias_fail_1] are empty")); } } @@ -289,6 +289,156 @@ public void testGetMappingsViewSuccess() throws IOException { assertEquals(2, unmappedFieldAliases.size()); } + public void testCreateMappings_withIndexPattern_success() throws IOException { + String indexName1 = "test_index_1"; + String indexName2 = "test_index_2"; + String indexPattern = "test_index*"; + + createIndex(indexName1, Settings.EMPTY, null); + createIndex(indexName2, Settings.EMPTY, null); + + client().performRequest(new Request("POST", "_refresh")); + + // Insert sample doc + String sampleDoc = "{" + + " \"netflow.source_ipv4_address\":\"10.50.221.10\"," + + " \"netflow.destination_transport_port\":1234," + + " \"netflow.destination_ipv4_address\":\"10.53.111.14\"," + + " \"netflow.source_transport_port\":4444" + + "}"; + + indexDoc(indexName1, "1", sampleDoc); + indexDoc(indexName2, "1", sampleDoc); + + client().performRequest(new Request("POST", "_refresh")); + + // Execute CreateMappingsAction to add alias mapping for index + Request request = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + request.setJsonEntity( + "{ \"index_name\":\"" + indexPattern + "\"," + + " \"rule_topic\":\"netflow\", " + + " \"partial\":true" + + "}" + ); + Response response = client().performRequest(request); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + } + + public void testCreateMappings_withIndexPattern_differentMappings_success() throws IOException { + String indexName1 = "test_index_1"; + String indexName2 = "test_index_2"; + String indexPattern = "test_index*"; + + createIndex(indexName1, Settings.EMPTY, null); + createIndex(indexName2, Settings.EMPTY, null); + + client().performRequest(new Request("POST", "_refresh")); + + // Insert sample docs + String sampleDoc1 = "{" + + " \"netflow.source_ipv4_address\":\"10.50.221.10\"," + + " \"netflow.destination_transport_port\":1234," + + " \"netflow.source_transport_port\":4444" + + "}"; + String sampleDoc2 = "{" + + " \"netflow.destination_transport_port\":1234," + + " \"netflow.destination_ipv4_address\":\"10.53.111.14\"" + + "}"; + indexDoc(indexName1, "1", sampleDoc1); + indexDoc(indexName2, "1", sampleDoc2); + + client().performRequest(new Request("POST", "_refresh")); + + // Execute CreateMappingsAction to add alias mapping for index + Request request = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + request.setJsonEntity( + "{ \"index_name\":\"" + indexPattern + "\"," + + " \"rule_topic\":\"netflow\", " + + " \"partial\":true" + + "}" + ); + Response response = client().performRequest(request); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + } + + public void testCreateMappings_withIndexPattern_oneNoMatches_success() throws IOException { + String indexName1 = "test_index_1"; + String indexName2 = "test_index_2"; + String indexPattern = "test_index*"; + + createIndex(indexName1, Settings.EMPTY, null); + createIndex(indexName2, Settings.EMPTY, null); + + client().performRequest(new Request("POST", "_refresh")); + + // Insert sample docs + String sampleDoc1 = "{" + + " \"netflow.source_ipv4_address\":\"10.50.221.10\"," + + " \"netflow.destination_transport_port\":1234," + + " \"netflow.source_transport_port\":4444" + + "}"; + String sampleDoc2 = "{" + + " \"netflow11.destination33_transport_port\":1234," + + " \"netflow11.destination33_ipv4_address\":\"10.53.111.14\"" + + "}"; + indexDoc(indexName1, "1", sampleDoc1); + indexDoc(indexName2, "1", sampleDoc2); + + client().performRequest(new Request("POST", "_refresh")); + + // Execute CreateMappingsAction to add alias mapping for index + Request request = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + request.setJsonEntity( + "{ \"index_name\":\"" + indexPattern + "\"," + + " \"rule_topic\":\"netflow\", " + + " \"partial\":true" + + "}" + ); + Response response = client().performRequest(request); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + } + + public void testCreateMappings_withIndexPattern_oneNoMappings_failure() throws IOException { + String indexName1 = "test_index_1"; + String indexName2 = "test_index_2"; + String indexPattern = "test_index*"; + + createIndex(indexName1, Settings.EMPTY, null); + createIndex(indexName2, Settings.EMPTY, null); + + client().performRequest(new Request("POST", "_refresh")); + + // Insert sample docs + String sampleDoc1 = "{" + + " \"netflow.source_ipv4_address\":\"10.50.221.10\"," + + " \"netflow.destination_transport_port\":1234," + + " \"netflow.source_transport_port\":4444" + + "}"; + indexDoc(indexName1, "1", sampleDoc1); + + client().performRequest(new Request("POST", "_refresh")); + + // Execute CreateMappingsAction to add alias mapping for index + Request request = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + request.setJsonEntity( + "{ \"index_name\":\"" + indexPattern + "\"," + + " \"rule_topic\":\"netflow\", " + + " \"partial\":true" + + "}" + ); + try { + client().performRequest(request); + fail("expected 500 failure!"); + } catch (ResponseException e) { + assertEquals(HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getResponse().getStatusLine().getStatusCode()); + } + + } + private void createSampleIndex(String indexName) throws IOException { String indexMapping = " \"properties\": {" + diff --git a/src/test/java/org/opensearch/securityanalytics/mapper/MapperServiceTests.java b/src/test/java/org/opensearch/securityanalytics/mapper/MapperServiceTests.java index e98486fdd..cde4be6dd 100644 --- a/src/test/java/org/opensearch/securityanalytics/mapper/MapperServiceTests.java +++ b/src/test/java/org/opensearch/securityanalytics/mapper/MapperServiceTests.java @@ -14,6 +14,7 @@ import org.opensearch.cluster.metadata.MappingMetadata; import org.opensearch.common.collect.ImmutableOpenMap; import org.opensearch.securityanalytics.action.GetMappingsViewResponse; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; @@ -59,7 +60,8 @@ public void onResponse(AcknowledgedResponse acknowledgedResponse) { @Override public void onFailure(Exception e) { - assertTrue(e.getMessage().equals("Alias mappings are missing path for alias: [srcport]")); + assertTrue(e instanceof SecurityAnalyticsException); + assertTrue(e.getCause().getMessage().equals("Alias mappings are missing path for alias: [srcport]")); } }); } @@ -99,7 +101,8 @@ public void onResponse(AcknowledgedResponse acknowledgedResponse) { @Override public void onFailure(Exception e) { - assertTrue(e.getMessage().contains("Duplicate field 'srcaddr'")); + assertTrue(e instanceof SecurityAnalyticsException); + assertTrue(e.getCause().getMessage().contains("Duplicate field 'srcaddr'")); } }); } diff --git a/src/test/java/org/opensearch/securityanalytics/mapper/MapperUtilsTests.java b/src/test/java/org/opensearch/securityanalytics/mapper/MapperUtilsTests.java index 8667da25b..bf093314a 100644 --- a/src/test/java/org/opensearch/securityanalytics/mapper/MapperUtilsTests.java +++ b/src/test/java/org/opensearch/securityanalytics/mapper/MapperUtilsTests.java @@ -13,6 +13,7 @@ import java.io.IOException; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; public class MapperUtilsTests extends OpenSearchTestCase { @@ -31,7 +32,7 @@ public void testValidateIndexMappingsMissingSome() throws IOException { MappingMetadata mappingMetadata = new MappingMetadata(MapperService.SINGLE_MAPPING_NAME, root); mappings.put("my_index", mappingMetadata); - List missingFields = MapperUtils.validateIndexMappings(mappings.build(), MapperTopicStore.aliasMappings("test123")); + List missingFields = MapperUtils.validateIndexMappings("my_index", mappingMetadata, MapperTopicStore.aliasMappings("test123")); assertEquals(3, missingFields.size()); } @@ -44,8 +45,8 @@ public void testValidateIndexMappingsEmptyMappings() throws IOException { MappingMetadata mappingMetadata = new MappingMetadata(MapperService.SINGLE_MAPPING_NAME, root); mappings.put("my_index", mappingMetadata); - IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> MapperUtils.validateIndexMappings(mappings.build(), MapperTopicStore.aliasMappings("test123"))); - assertTrue(e.getMessage().contains("Index mappings are empty")); + IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> MapperUtils.validateIndexMappings("my_index", mappingMetadata, MapperTopicStore.aliasMappings("test123"))); + assertTrue(e.getMessage().contains(String.format(Locale.getDefault(), "Mappings for index [%s] are empty", "my_index"))); } public void testValidateIndexMappingsNoMissing() throws IOException { @@ -60,7 +61,7 @@ public void testValidateIndexMappingsNoMissing() throws IOException { MappingMetadata mappingMetadata = new MappingMetadata(MapperService.SINGLE_MAPPING_NAME, root); mappings.put("my_index", mappingMetadata); - List missingFields = MapperUtils.validateIndexMappings(mappings.build(), MapperTopicStore.aliasMappings("test123")); + List missingFields = MapperUtils.validateIndexMappings("my_index", mappingMetadata, MapperTopicStore.aliasMappings("test123")); assertEquals(0, missingFields.size()); } diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java index f0aad2e8b..180d670bf 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/DetectorRestApiIT.java @@ -17,6 +17,7 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.client.Request; import org.opensearch.client.Response; +import org.opensearch.common.settings.Settings; import org.opensearch.commons.alerting.model.Monitor.MonitorType; import org.opensearch.rest.RestStatus; import org.opensearch.search.SearchHit; @@ -144,7 +145,7 @@ public void testCreateDetectorWithoutRules() throws IOException { " }\n" + " }\n" + "}"; - SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()), request, true); + SearchResponse response = executeSearchAndGetResponse(DetectorMonitorConfig.getRuleIndex(randomDetectorType()) + "*", request, true); Assert.assertEquals(0, response.getHits().getTotalHits().value); String createdId = responseBody.get("_id").toString(); @@ -458,7 +459,7 @@ public void testUpdateADetector() throws IOException { } @SuppressWarnings("unchecked") - public void testDeletingADetector() throws IOException { + public void testDeletingADetector_single_ruleTopicIndex() throws IOException { String index = createTestIndex(randomIndex(), windowsIndexMapping()); // Execute CreateMappingsAction to add alias mapping for index @@ -473,20 +474,14 @@ public void testDeletingADetector() throws IOException { Response response = client().performRequest(createMappingRequest); assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); - - Detector detector = randomDetector(getRandomPrePackagedRules()); - - Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); - Assert.assertEquals("Create detector failed", RestStatus.CREATED, restStatus(createResponse)); - - Map responseBody = asMap(createResponse); - - String createdId = responseBody.get("_id").toString(); + // Create detector #1 of type test_windows + Detector detector1 = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of()))); + String detectorId1 = createDetector(detector1); String request = "{\n" + " \"query\" : {\n" + " \"match\":{\n" + - " \"_id\": \"" + createdId + "\"\n" + + " \"_id\": \"" + detectorId1 + "\"\n" + " }\n" + " }\n" + "}"; @@ -495,13 +490,134 @@ public void testDeletingADetector() throws IOException { String monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); - Response deleteResponse = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + createdId, Collections.emptyMap(), null); + indexDoc(index, "1", randomDoc()); + + Response executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + Map executeResults = entityAsMap(executeResponse); + + int noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + // Create detector #2 of type windows + Detector detector2 = randomDetectorWithTriggers(getRandomPrePackagedRules(), List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of()))); + String detectorId2 = createDetector(detector2); + + request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId2 + "\"\n" + + " }\n" + + " }\n" + + "}"; + hits = executeSearch(Detector.DETECTORS_INDEX, request); + hit = hits.get(0); + + monitorId = ((List) ((Map) hit.getSourceAsMap().get("detector")).get("monitor_id")).get(0); + + indexDoc(index, "1", randomDoc()); + + executeResponse = executeAlertingMonitor(monitorId, Collections.emptyMap()); + executeResults = entityAsMap(executeResponse); + noOfSigmaRuleMatches = ((List>) ((Map) executeResults.get("input_results")).get("results")).get(0).size(); + Assert.assertEquals(5, noOfSigmaRuleMatches); + + Response deleteResponse = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId1, Collections.emptyMap(), null); Assert.assertEquals("Delete detector failed", RestStatus.OK, restStatus(deleteResponse)); + // We deleted 1 detector, but 1 detector with same type exists, so we expect queryIndex to be present + Assert.assertTrue(doesIndexExist(String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries-000001", "test_windows"))); - Assert.assertFalse(alertingMonitorExists(monitorId)); + deleteResponse = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId2, Collections.emptyMap(), null); + Assert.assertEquals("Delete detector failed", RestStatus.OK, restStatus(deleteResponse)); + // We deleted all detectors of type windows, so we expect that queryIndex is deleted + Assert.assertFalse(doesIndexExist(String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries-000001", "test_windows"))); - Assert.assertFalse(doesIndexExist(String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries", "windows"))); + request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId1 + "\"\n" + + " }\n" + + " }\n" + + "}"; + hits = executeSearch(Detector.DETECTORS_INDEX, request); + Assert.assertEquals(0, hits.size()); + + request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId2 + "\"\n" + + " }\n" + + " }\n" + + "}"; + hits = executeSearch(Detector.DETECTORS_INDEX, request); + Assert.assertEquals(0, hits.size()); + } + + public void testDeletingADetector_oneDetectorType_multiple_ruleTopicIndex() throws IOException { + String index1 = "test_index_1"; + createIndex(index1, Settings.EMPTY); + String index2 = "test_index_2"; + createIndex(index2, Settings.EMPTY); + // Insert doc with 900 fields to update mappings too + String doc = createDocumentWithNFields(900); + indexDoc(index1, "1", doc); + indexDoc(index2, "1", doc); + + // Create detector #1 of type test_windows + Detector detector1 = randomDetectorWithTriggers( + getRandomPrePackagedRules(), + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of())), + List.of(index1) + ); + String detectorId1 = createDetector(detector1); + + // Create detector #2 of type test_windows + Detector detector2 = randomDetectorWithTriggers( + getRandomPrePackagedRules(), + List.of(new DetectorTrigger(null, "test-trigger", "1", List.of(randomDetectorType()), List.of(), List.of(), List.of(), List.of())), + List.of(index2) + ); + + String detectorId2 = createDetector(detector2); + + Assert.assertTrue(doesIndexExist(".opensearch-sap-test_windows-detectors-queries-000001")); + Assert.assertTrue(doesIndexExist(".opensearch-sap-test_windows-detectors-queries-000002")); + + // Check if both query indices have proper settings applied from index template + Map settings = getIndexSettingsAsMap(".opensearch-sap-test_windows-detectors-queries-000001"); + assertTrue(settings.containsKey("index.analysis.char_filter.rule_ws_filter.pattern")); + assertTrue(settings.containsKey("index.hidden")); + settings = getIndexSettingsAsMap(".opensearch-sap-test_windows-detectors-queries-000002"); + assertTrue(settings.containsKey("index.analysis.char_filter.rule_ws_filter.pattern")); + assertTrue(settings.containsKey("index.hidden")); + Response deleteResponse = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId1, Collections.emptyMap(), null); + Assert.assertEquals("Delete detector failed", RestStatus.OK, restStatus(deleteResponse)); + // We deleted 1 detector, but 1 detector with same type exists, so we expect queryIndex to be present + Assert.assertFalse(doesIndexExist(String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries-000001", "test_windows"))); + Assert.assertTrue(doesIndexExist(String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries-000002", "test_windows"))); + + deleteResponse = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + detectorId2, Collections.emptyMap(), null); + Assert.assertEquals("Delete detector failed", RestStatus.OK, restStatus(deleteResponse)); + // We deleted all detectors of type windows, so we expect that queryIndex is deleted + Assert.assertFalse(doesIndexExist(String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries-000001", "test_windows"))); + Assert.assertFalse(doesIndexExist(String.format(Locale.getDefault(), ".opensearch-sap-%s-detectors-queries-000002", "test_windows"))); + + String request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId1 + "\"\n" + + " }\n" + + " }\n" + + "}"; + List hits = executeSearch(Detector.DETECTORS_INDEX, request); + Assert.assertEquals(0, hits.size()); + + request = "{\n" + + " \"query\" : {\n" + + " \"match\":{\n" + + " \"_id\": \"" + detectorId2 + "\"\n" + + " }\n" + + " }\n" + + "}"; hits = executeSearch(Detector.DETECTORS_INDEX, request); Assert.assertEquals(0, hits.size()); }