From 541b99b764292dd732b9f6a61aaeacec0a93a139 Mon Sep 17 00:00:00 2001 From: Megha Goyal <56077967+goyamegh@users.noreply.github.com> Date: Thu, 18 Jan 2024 17:31:43 -0800 Subject: [PATCH] Allow deletion of custom log type if custom rule index is missing (#767) (#814) * Allow deletion of custom log type if custom rule index is missing * Allow custom log type name to be updated when custom rule index is missing * Adding changes for detector index missing * Update log type when detector index is missing --------- Signed-off-by: Megha Goyal --- .../TransportDeleteCustomLogTypeAction.java | 129 +++++----- .../transport/TransportDeleteRuleAction.java | 1 + .../TransportIndexCustomLogTypeAction.java | 227 +++++++----------- .../securityanalytics/util/RuleIndices.java | 19 ++ .../resthandler/CustomLogTypeRestApiIT.java | 212 +++++++++++++++- 5 files changed, 383 insertions(+), 205 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportDeleteCustomLogTypeAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportDeleteCustomLogTypeAction.java index b99cd6a43..cb4ec4a19 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportDeleteCustomLogTypeAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportDeleteCustomLogTypeAction.java @@ -26,9 +26,10 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; import org.opensearch.commons.authuser.User; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; -import org.opensearch.core.rest.RestStatus; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.securityanalytics.action.DeleteCustomLogTypeAction; import org.opensearch.securityanalytics.action.DeleteCustomLogTypeRequest; @@ -36,16 +37,15 @@ import org.opensearch.securityanalytics.logtype.LogTypeService; import org.opensearch.securityanalytics.model.CustomLogType; import org.opensearch.securityanalytics.model.Detector; -import org.opensearch.securityanalytics.model.Rule; import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; import org.opensearch.securityanalytics.util.CustomLogTypeIndices; import org.opensearch.securityanalytics.util.DetectorIndices; +import org.opensearch.securityanalytics.util.RuleIndices; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; -import java.io.IOException; import java.util.Locale; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; @@ -65,6 +65,8 @@ public class TransportDeleteCustomLogTypeAction extends HandledTransportAction() { - @Override - public void onResponse(SearchResponse response) { - if (response.isTimedOut()) { - onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Search request timed out. Log Type with id %s cannot be deleted", logType.getId()), RestStatus.REQUEST_TIMEOUT)); - return; - } - - if (response.getHits().getTotalHits().value > 0) { - onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted because active rules exist", logType.getId()), RestStatus.BAD_REQUEST)); - return; - } - - DeleteRequest deleteRequest = new DeleteRequest(LogTypeService.LOG_TYPE_INDEX, logType.getId()) - .setRefreshPolicy(request.getRefreshPolicy()) - .timeout(indexTimeout); - - client.delete(deleteRequest, new ActionListener<>() { - @Override - public void onResponse(DeleteResponse response) { - if (response.status() != RestStatus.OK) { - onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted", logType.getId()), RestStatus.INTERNAL_SERVER_ERROR)); - } - onOperation(response); - } - - @Override - public void onFailure(Exception e) { - onFailures(e); - } - }); - } - - @Override - public void onFailure(Exception e) { - onFailures(e); - } - }); + checkRuleIndexAndDeleteLogType(logType); } @Override @@ -229,25 +198,62 @@ public void onFailure(Exception e) { } }); } else { - DeleteRequest deleteRequest = new DeleteRequest(LogTypeService.LOG_TYPE_INDEX, logType.getId()) - .setRefreshPolicy(request.getRefreshPolicy()) - .timeout(indexTimeout); + checkRuleIndexAndDeleteLogType(logType); + } + } - client.delete(deleteRequest, new ActionListener<>() { + void checkRuleIndexAndDeleteLogType(CustomLogType logType) { + if(ruleIndices.ruleIndexExists(false)) { + ruleIndices.searchRules(logType.getName(), new ActionListener<>() { @Override - public void onResponse(DeleteResponse response) { - if (response.status() != RestStatus.OK) { - onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted", logType.getId()), RestStatus.INTERNAL_SERVER_ERROR)); + public void onResponse(SearchResponse response) { + if (response.isTimedOut()) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Search request timed out. Log Type with id %s cannot be deleted", logType.getId()), RestStatus.REQUEST_TIMEOUT)); + return; } - onOperation(response); + + if (response.getHits().getTotalHits().value > 0) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted because active rules exist", logType.getId()), RestStatus.BAD_REQUEST)); + return; + } + deleteLogType(logType); } @Override public void onFailure(Exception e) { - onFailures(e); + if (e instanceof IndexNotFoundException) { + // let log type deletion to go through if the rule index is missing + deleteLogType(logType); + } else { + onFailures(e); + } } }); + } else { + log.warn("Custom rule index missing, allowing deletion of custom log type {} to go through", logType.getId()); + deleteLogType(logType); } + } + + private void deleteLogType(CustomLogType logType) { + DeleteRequest deleteRequest = new DeleteRequest(LogTypeService.LOG_TYPE_INDEX, logType.getId()) + .setRefreshPolicy(request.getRefreshPolicy()) + .timeout(indexTimeout); + + client.delete(deleteRequest, new ActionListener<>() { + @Override + public void onResponse(DeleteResponse response) { + if (response.status() != RestStatus.OK) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted", logType.getId()), RestStatus.INTERNAL_SERVER_ERROR)); + } + onOperation(response); + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); } private void searchDetectors(String logTypeName, ActionListener listener) { @@ -267,23 +273,6 @@ private void searchDetectors(String logTypeName, ActionListener client.search(searchRequest, listener); } - private void searchRules(String logTypeName, ActionListener listener) { - QueryBuilder queryBuilder = - QueryBuilders.nestedQuery("rule", - QueryBuilders.boolQuery().must( - QueryBuilders.matchQuery("rule.category", logTypeName) - ), ScoreMode.Avg); - - SearchRequest searchRequest = new SearchRequest(Rule.CUSTOM_RULES_INDEX) - .source(new SearchSourceBuilder() - .seqNoAndPrimaryTerm(true) - .version(true) - .query(queryBuilder) - .size(0)); - - client.search(searchRequest, listener); - } - private void onOperation(DeleteResponse response) { this.response.set(response); if (counter.compareAndSet(false, true)) { @@ -292,7 +281,7 @@ private void onOperation(DeleteResponse response) { } private void onFailures(Exception t) { - log.error(String.format(Locale.ROOT, "Failed to delete detector")); + log.error(String.format(Locale.ROOT, "Failed to delete log type"), t); if (counter.compareAndSet(false, true)) { finishHim(null, t); } diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportDeleteRuleAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportDeleteRuleAction.java index 4d9adbb7d..7e0d70e0e 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportDeleteRuleAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportDeleteRuleAction.java @@ -227,6 +227,7 @@ private void deleteRule(String ruleId) { new DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE) .source(Rule.CUSTOM_RULES_INDEX) .filter(QueryBuilders.matchQuery("_id", ruleId)) + .refresh(true) .execute(new ActionListener<>() { @Override public void onResponse(BulkByScrollResponse response) { diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexCustomLogTypeAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexCustomLogTypeAction.java index a1bfdaa89..f98135fa6 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexCustomLogTypeAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexCustomLogTypeAction.java @@ -28,6 +28,7 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.commons.authuser.User; import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.index.IndexNotFoundException; import org.opensearch.index.query.MatchQueryBuilder; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; @@ -43,11 +44,11 @@ import org.opensearch.securityanalytics.logtype.LogTypeService; import org.opensearch.securityanalytics.model.CustomLogType; import org.opensearch.securityanalytics.model.Detector; -import org.opensearch.securityanalytics.model.Rule; import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; import org.opensearch.securityanalytics.util.CustomLogTypeIndices; import org.opensearch.securityanalytics.util.DetectorIndices; import org.opensearch.securityanalytics.util.IndexUtils; +import org.opensearch.securityanalytics.util.RuleIndices; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; @@ -72,6 +73,8 @@ public class TransportIndexCustomLogTypeAction extends HandledTransportAction sourceMap = response.getHits().getHits()[0].getSourceAsMap(); - CustomLogType existingLogType = new CustomLogType(sourceMap); - existingLogType.setId(request.getCustomLogType().getId()); - existingLogType.setVersion(request.getCustomLogType().getVersion()); - - if (existingLogType.getSource().equals("Sigma")) { - onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be updated because source is sigma", logTypeId), RestStatus.BAD_REQUEST)); - } - if (!existingLogType.getName().equals(request.getCustomLogType().getName())) { - - if (detectorIndices.detectorIndexExists()) { - searchDetectors(existingLogType.getName(), new ActionListener<>() { - @Override - public void onResponse(SearchResponse response) { - if (response.isTimedOut()) { - onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Search request timed out. Log Type with id %s cannot be updated", logTypeId), RestStatus.REQUEST_TIMEOUT)); - return; - } - - if (response.getHits().getTotalHits().value > 0) { - onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Name of Log Type with id %s cannot be updated because active detectors exist", logTypeId), RestStatus.BAD_REQUEST)); - return; - } - - searchRules(existingLogType.getName(), new ActionListener<>() { - @Override - public void onResponse(SearchResponse response) { - if (response.isTimedOut()) { - onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Search request timed out. Log Type with id %s cannot be updated", logTypeId), RestStatus.REQUEST_TIMEOUT)); - return; - } - - if (response.getHits().getTotalHits().value > 0) { - onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Name of Log Type with id %s cannot be updated because active rules exist", logTypeId), RestStatus.BAD_REQUEST)); - return; - } - - try { - request.getCustomLogType().setTags(existingLogType.getTags()); - IndexRequest indexRequest = new IndexRequest(LogTypeService.LOG_TYPE_INDEX) - .setRefreshPolicy(request.getRefreshPolicy()) - .source(request.getCustomLogType().toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) - .id(request.getLogTypeId()) - .timeout(indexTimeout); - - client.index(indexRequest, new ActionListener<>() { - @Override - public void onResponse(IndexResponse response) { - if (response.status() != RestStatus.OK) { - onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be updated", logTypeId), RestStatus.INTERNAL_SERVER_ERROR)); - } - onOperation(response, request.getCustomLogType()); - } - - @Override - public void onFailure(Exception e) { - onFailures(e); - } - }); - } catch (IOException e) { - onFailures(e); - } - } - - @Override - public void onFailure(Exception e) { - onFailures(e); - } - }); - } + Map sourceMap = response.getHits().getHits()[0].getSourceAsMap(); + CustomLogType existingLogType = new CustomLogType(sourceMap); + existingLogType.setId(request.getCustomLogType().getId()); + existingLogType.setVersion(request.getCustomLogType().getVersion()); - @Override - public void onFailure(Exception e) { - onFailures(e); - } - }); - } else { - request.getCustomLogType().setTags(existingLogType.getTags()); - IndexRequest indexRequest = new IndexRequest(LogTypeService.LOG_TYPE_INDEX) - .setRefreshPolicy(request.getRefreshPolicy()) - .source(request.getCustomLogType().toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) - .id(request.getLogTypeId()) - .timeout(indexTimeout); - - client.index(indexRequest, new ActionListener<>() { - @Override - public void onResponse(IndexResponse response) { - if (response.status() != RestStatus.OK) { - onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be updated", logTypeId), RestStatus.INTERNAL_SERVER_ERROR)); - } - - request.getCustomLogType().setId(response.getId()); - onOperation(response, request.getCustomLogType()); - } + if (existingLogType.getSource().equals("Sigma")) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be updated because source is sigma", logTypeId), RestStatus.BAD_REQUEST)); + } + if (!existingLogType.getName().equals(request.getCustomLogType().getName())) { - @Override - public void onFailure(Exception e) { - onFailures(e); - } - }); - } - } else { - request.getCustomLogType().setTags(existingLogType.getTags()); - IndexRequest indexRequest = new IndexRequest(LogTypeService.LOG_TYPE_INDEX) - .setRefreshPolicy(request.getRefreshPolicy()) - .source(request.getCustomLogType().toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) - .id(request.getLogTypeId()) - .timeout(indexTimeout); - - client.index(indexRequest, new ActionListener<>() { + if (detectorIndices.detectorIndexExists()) { + searchDetectors(existingLogType.getName(), new ActionListener<>() { @Override - public void onResponse(IndexResponse response) { - if (response.status() != RestStatus.OK) { - onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be updated", logTypeId), RestStatus.INTERNAL_SERVER_ERROR)); + public void onResponse(SearchResponse response) { + if (response.isTimedOut()) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Search request timed out. Log Type with id %s cannot be updated", logTypeId), RestStatus.REQUEST_TIMEOUT)); + return; } - request.getCustomLogType().setId(response.getId()); - onOperation(response, request.getCustomLogType()); + if (response.getHits().getTotalHits().value > 0) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Name of Log Type with id %s cannot be updated because active detectors exist", logTypeId), RestStatus.BAD_REQUEST)); + return; + } + checkRuleIndexAndUpdateLogType(existingLogType, logTypeId); } @Override @@ -354,9 +264,11 @@ public void onFailure(Exception e) { onFailures(e); } }); + } else { + checkRuleIndexAndUpdateLogType(existingLogType, logTypeId); } - } catch (IOException e) { - onFailures(e); + } else { + updateLogType(existingLogType, logTypeId); } } @@ -455,6 +367,70 @@ public void onFailure(Exception e) { } } + void checkRuleIndexAndUpdateLogType(CustomLogType existingLogType, String logTypeId) { + if (ruleIndices.ruleIndexExists(false)) { + ruleIndices.searchRules(existingLogType.getName(), new ActionListener<>() { + @Override + public void onResponse(SearchResponse response) { + if (response.isTimedOut()) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Search request timed out. Log Type with id %s cannot be updated", logTypeId), RestStatus.REQUEST_TIMEOUT)); + return; + } + + if (response.getHits().getTotalHits().value > 0) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Name of Log Type with id %s cannot be updated because active rules exist", logTypeId), RestStatus.BAD_REQUEST)); + return; + } + updateLogType(existingLogType, logTypeId); + } + + @Override + public void onFailure(Exception e) { + if (e instanceof IndexNotFoundException) { + // let log type update if the rule index is missing + updateLogType(existingLogType, logTypeId); + } else { + onFailures(e); + } + } + }); + } else { + log.warn("Custom rule index missing, allowing update of custom log type {} to go through", logTypeId); + updateLogType(existingLogType, logTypeId); + } + + } + + private void updateLogType(CustomLogType existingLogType, String logTypeId) { + try { + request.getCustomLogType().setTags(existingLogType.getTags()); + IndexRequest indexRequest = new IndexRequest(LogTypeService.LOG_TYPE_INDEX) + .setRefreshPolicy(request.getRefreshPolicy()) + .source(request.getCustomLogType().toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + .id(logTypeId) + .timeout(indexTimeout); + + client.index(indexRequest, new ActionListener<>() { + @Override + public void onResponse(IndexResponse response) { + if (response.status() != RestStatus.OK) { + onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be updated", logTypeId), RestStatus.INTERNAL_SERVER_ERROR)); + } + + request.getCustomLogType().setId(response.getId()); + onOperation(response, request.getCustomLogType()); + } + + @Override + public void onFailure(Exception e) { + onFailures(e); + } + }); + } catch (IOException e) { + onFailures(e); + } + } + private void searchLogTypes(String logTypeId, ActionListener listener) { QueryBuilder queryBuilder = QueryBuilders.matchQuery("_id", logTypeId); SearchRequest searchRequest = new SearchRequest(LogTypeService.LOG_TYPE_INDEX) @@ -483,23 +459,6 @@ private void searchDetectors(String logTypeName, ActionListener client.search(searchRequest, listener); } - private void searchRules(String logTypeName, ActionListener listener) { - QueryBuilder queryBuilder = - QueryBuilders.nestedQuery("rule", - QueryBuilders.boolQuery().must( - QueryBuilders.matchQuery("rule.category", logTypeName) - ), ScoreMode.Avg); - - SearchRequest searchRequest = new SearchRequest(Rule.CUSTOM_RULES_INDEX) - .source(new SearchSourceBuilder() - .seqNoAndPrimaryTerm(true) - .version(true) - .query(queryBuilder) - .size(0)); - - client.search(searchRequest, listener); - } - private void onOperation(IndexResponse response, CustomLogType logType) { this.response.set(response); if (counter.compareAndSet(false, true)) { diff --git a/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java b/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java index c2dda63a6..94a98106b 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java +++ b/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java @@ -8,6 +8,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.lucene.search.join.ScoreMode; import org.opensearch.OpenSearchStatusException; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.create.CreateIndexResponse; @@ -33,6 +34,7 @@ import org.opensearch.core.rest.RestStatus; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.reindex.BulkByScrollResponse; import org.opensearch.index.reindex.DeleteByQueryAction; @@ -223,6 +225,23 @@ public void countRules(ActionListener listener) { client.search(request, listener); } + public void searchRules(String logTypeName, ActionListener listener) { + QueryBuilder queryBuilder = + QueryBuilders.nestedQuery("rule", + QueryBuilders.boolQuery().must( + QueryBuilders.matchQuery("rule.category", logTypeName) + ), ScoreMode.Avg); + + SearchRequest searchRequest = new SearchRequest(Rule.CUSTOM_RULES_INDEX) + .source(new SearchSourceBuilder() + .seqNoAndPrimaryTerm(true) + .version(true) + .query(queryBuilder) + .size(0)); + + client.search(searchRequest, listener); + } + private List getRules(List listOfRules) { List rules = new ArrayList<>(); diff --git a/src/test/java/org/opensearch/securityanalytics/resthandler/CustomLogTypeRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/resthandler/CustomLogTypeRestApiIT.java index 68d7d6635..2cce021dd 100644 --- a/src/test/java/org/opensearch/securityanalytics/resthandler/CustomLogTypeRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/resthandler/CustomLogTypeRestApiIT.java @@ -11,6 +11,7 @@ import org.apache.http.entity.StringEntity; import org.apache.http.message.BasicHeader; import org.junit.Assert; +import org.opensearch.action.admin.indices.refresh.RefreshRequest; import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; @@ -32,6 +33,8 @@ import java.util.Map; import static org.opensearch.securityanalytics.TestHelpers.*; +import static org.opensearch.securityanalytics.logtype.LogTypeService.LOG_TYPE_INDEX; +import static org.opensearch.securityanalytics.model.Rule.CUSTOM_RULES_INDEX; public class CustomLogTypeRestApiIT extends SecurityAnalyticsRestTestCase { @@ -387,6 +390,109 @@ public void testEditACustomLogTypeNameFailsAsCustomRuleExist() throws IOExceptio }); } + @SuppressWarnings("unchecked") + public void testEditACustomLogTypeNameWhenCustomRuleIndexMissing() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + CustomLogType customLogType = TestHelpers.randomCustomLogType(null, null, null, "Custom"); + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI, Collections.emptyMap(), toHttpEntity(customLogType)); + Assert.assertEquals("Create custom log type failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + String logTypeId = responseBody.get("_id").toString(); + Assert.assertEquals(customLogType.getDescription(), ((Map) responseBody.get("logType")).get("description")); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + customLogType.getName() + "\", " + + " \"partial\":true, " + + " \"alias_mappings\":{}" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + String rule = randomRule(); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", customLogType.getName()), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + String createdId = responseBody.get("_id").toString(); + + DetectorInput input = new DetectorInput("custom log type detector for security analytics", List.of(index), List.of(new DetectorRule(createdId)), + List.of()); + Detector detector = randomDetectorWithInputs(List.of(input), customLogType.getName()); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector successful", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + createdId = responseBody.get("_id").toString(); + + Response deleteResponse = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + createdId, Collections.emptyMap(), null); + Assert.assertEquals("Delete detector successful", RestStatus.OK, restStatus(deleteResponse)); + + makeRequest(client(), "DELETE", "/.opensearch-sap-custom-rules-config", Collections.emptyMap(), new StringEntity("")); + + customLogType = TestHelpers.randomCustomLogType("test", null, "Access Management", "Custom"); + Response updatedResponse = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI + "/" + logTypeId, Collections.emptyMap(), toHttpEntity(customLogType)); + Assert.assertEquals("Update custom log type successful", RestStatus.OK, restStatus(updatedResponse)); + + responseBody = asMap(updatedResponse); + Assert.assertEquals(customLogType.getCategory(), ((Map) responseBody.get("logType")).get("category")); + } + + public void testEditACustomLogTypeNameWhenDetectorIndexMissing() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + CustomLogType customLogType = TestHelpers.randomCustomLogType(null, null, null, "Custom"); + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI, Collections.emptyMap(), toHttpEntity(customLogType)); + Assert.assertEquals("Create custom log type failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + String logTypeId = responseBody.get("_id").toString(); + Assert.assertEquals(customLogType.getDescription(), ((Map) responseBody.get("logType")).get("description")); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + customLogType.getName() + "\", " + + " \"partial\":true, " + + " \"alias_mappings\":{}" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + String rule = randomRule(); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", customLogType.getName()), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + String createdId = responseBody.get("_id").toString(); + + Response deleteResponse = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.RULE_BASE_URI + "/" + createdId, Collections.emptyMap(), new StringEntity("")); + Assert.assertEquals("Delete rule successful", RestStatus.OK, restStatus(deleteResponse)); + + customLogType = TestHelpers.randomCustomLogType("test", null, "Access Management", "Custom"); + Response updatedResponse = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI + "/" + logTypeId, Collections.emptyMap(), toHttpEntity(customLogType)); + Assert.assertEquals("Update custom log type successful", RestStatus.OK, restStatus(updatedResponse)); + + responseBody = asMap(updatedResponse); + Assert.assertEquals(customLogType.getCategory(), ((Map) responseBody.get("logType")).get("category")); + } + @SuppressWarnings("unchecked") public void testEditACustomLogTypeName() throws IOException, InterruptedException { String index = createTestIndex(randomIndex(), windowsIndexMapping()); @@ -437,7 +543,6 @@ public void testEditACustomLogTypeName() throws IOException, InterruptedExceptio deleteResponse = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.RULE_BASE_URI + "/" + ruleId, Collections.emptyMap(), null); Assert.assertEquals("Delete rule failed", RestStatus.OK, restStatus(deleteResponse)); - Thread.sleep(5000); CustomLogType updatedCustomLogType = TestHelpers.randomCustomLogType("updated_name", null, null, "Custom"); Response updatedResponse = makeRequest(client(), "PUT", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI + "/" + logTypeId, Collections.emptyMap(), toHttpEntity(updatedCustomLogType)); @@ -547,6 +652,111 @@ public void testDeleteCustomLogTypeFailsAsDetectorExist() throws IOException { }); } + @SuppressWarnings("unchecked") + public void testDeleteCustomLogTypeWithDetectorIndexMissing() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + CustomLogType customLogType = TestHelpers.randomCustomLogType(null, null, null, "Custom"); + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI, Collections.emptyMap(), toHttpEntity(customLogType)); + Assert.assertEquals("Create custom log type successful", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + String logTypeId = responseBody.get("_id").toString(); + Assert.assertEquals(customLogType.getDescription(), ((Map) responseBody.get("logType")).get("description")); + + String rule = randomRule(); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", customLogType.getName()), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule successful", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + String ruleId = responseBody.get("_id").toString(); + + Response deleteResponse = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.RULE_BASE_URI + "/" + ruleId, Collections.emptyMap(), new StringEntity("")); + Assert.assertEquals("Delete rule successful", RestStatus.OK, restStatus(deleteResponse)); + + String request = "{\n" + + " \"query\": {\n" + + " \"nested\": {\n" + + " \"path\": \"rule\",\n" + + " \"query\": {\n" + + " \"bool\": {\n" + + " \"must\": [\n" + + " { \"match\": {\"rule.category\": \"" + customLogType.getName() + "\"}}\n" + + " ]\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; + + Response searchResponse = makeRequest(client(), "POST", String.format(Locale.getDefault(), "%s/_search", SecurityAnalyticsPlugin.RULE_BASE_URI), Collections.singletonMap("pre_packaged", "false"), + new StringEntity(request), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Searching rules successful", RestStatus.OK, restStatus(searchResponse)); + + responseBody = asMap(searchResponse); + Assert.assertEquals(0, ((Map) ((Map) responseBody.get("hits")).get("total")).get("value")); + + deleteResponse = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI + "/" + logTypeId, Collections.emptyMap(), new StringEntity("")); + Assert.assertEquals("Delete custom log type successful", RestStatus.OK, restStatus(deleteResponse)); + } + + @SuppressWarnings("unchecked") + public void testDeleteCustomLogTypeWithRuleIndexMissing() throws IOException { + String index = createTestIndex(randomIndex(), windowsIndexMapping()); + + CustomLogType customLogType = TestHelpers.randomCustomLogType(null, null, null, "Custom"); + Response createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI, Collections.emptyMap(), toHttpEntity(customLogType)); + Assert.assertEquals("Create custom log type failed", RestStatus.CREATED, restStatus(createResponse)); + + Map responseBody = asMap(createResponse); + String logTypeId = responseBody.get("_id").toString(); + Assert.assertEquals(customLogType.getDescription(), ((Map) responseBody.get("logType")).get("description")); + + // Execute CreateMappingsAction to add alias mapping for index + Request createMappingRequest = new Request("POST", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + // both req params and req body are supported + createMappingRequest.setJsonEntity( + "{ \"index_name\":\"" + index + "\"," + + " \"rule_topic\":\"" + customLogType.getName() + "\", " + + " \"partial\":true, " + + " \"alias_mappings\":{}" + + "}" + ); + + Response response = client().performRequest(createMappingRequest); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + String rule = randomRule(); + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.RULE_BASE_URI, Collections.singletonMap("category", customLogType.getName()), + new StringEntity(rule), new BasicHeader("Content-Type", "application/json")); + Assert.assertEquals("Create rule failed", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + String createdId = responseBody.get("_id").toString(); + + DetectorInput input = new DetectorInput("custom log type detector for security analytics", List.of(index), List.of(new DetectorRule(createdId)), + List.of()); + Detector detector = randomDetectorWithInputs(List.of(input), customLogType.getName()); + + + createResponse = makeRequest(client(), "POST", SecurityAnalyticsPlugin.DETECTOR_BASE_URI, Collections.emptyMap(), toHttpEntity(detector)); + Assert.assertEquals("Create detector successful", RestStatus.CREATED, restStatus(createResponse)); + + responseBody = asMap(createResponse); + createdId = responseBody.get("_id").toString(); + + Response deleteResponse = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.DETECTOR_BASE_URI + "/" + createdId, Collections.emptyMap(), null); + Assert.assertEquals("Delete detector successful", RestStatus.OK, restStatus(deleteResponse)); + + makeRequest(client(), "DELETE", "/.opensearch-sap-custom-rules-config", Collections.emptyMap(), new StringEntity("")); + + deleteResponse = makeRequest(client(), "DELETE", SecurityAnalyticsPlugin.CUSTOM_LOG_TYPE_URI + "/" + logTypeId, Collections.emptyMap(), new StringEntity("")); + Assert.assertEquals("Delete custom log type successful", RestStatus.OK, restStatus(deleteResponse)); + } + @SuppressWarnings("unchecked") public void testDeleteCustomLogTypeFailsAsRulesExist() throws IOException { String index = createTestIndex(randomIndex(), windowsIndexMapping());