Allow deletion of custom log type if custom rule index is missing (#767)

src/main/java/org/opensearch/securityanalytics/transport/TransportDeleteCustomLogTypeAction.java

@@ -26,26 +26,26 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.commons.authuser.User;
+import org.opensearch.core.rest.RestStatus;
+import org.opensearch.index.IndexNotFoundException;
 import org.opensearch.index.query.QueryBuilder;
 import org.opensearch.index.query.QueryBuilders;
-import org.opensearch.core.rest.RestStatus;
 import org.opensearch.search.builder.SearchSourceBuilder;
 import org.opensearch.securityanalytics.action.DeleteCustomLogTypeAction;
 import org.opensearch.securityanalytics.action.DeleteCustomLogTypeRequest;
 import org.opensearch.securityanalytics.action.DeleteCustomLogTypeResponse;
 import org.opensearch.securityanalytics.logtype.LogTypeService;
 import org.opensearch.securityanalytics.model.CustomLogType;
 import org.opensearch.securityanalytics.model.Detector;
-import org.opensearch.securityanalytics.model.Rule;
 import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings;
 import org.opensearch.securityanalytics.util.CustomLogTypeIndices;
 import org.opensearch.securityanalytics.util.DetectorIndices;
+import org.opensearch.securityanalytics.util.RuleIndices;
 import org.opensearch.securityanalytics.util.SecurityAnalyticsException;
 import org.opensearch.tasks.Task;
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.transport.TransportService;
 
-import java.io.IOException;
 import java.util.Locale;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -65,6 +65,8 @@
 
     private final DetectorIndices detectorIndices;
 
+    private final RuleIndices ruleIndices;
+
     private final CustomLogTypeIndices customLogTypeIndices;
 
     private volatile Boolean filterByEnabled;
@@ -77,6 +79,7 @@
                                               ActionFilters actionFilters,
                                               ClusterService clusterService,
                                               DetectorIndices detectorIndices,
+                                              RuleIndices ruleIndices,
                                               CustomLogTypeIndices customLogTypeIndices,
                                               Settings settings,
                                               ThreadPool threadPool) {
@@ -86,6 +89,7 @@
         this.threadPool = threadPool;
         this.settings = settings;
         this.detectorIndices = detectorIndices;
+        this.ruleIndices = ruleIndices;
         this.customLogTypeIndices = customLogTypeIndices;
         this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings);
         this.indexTimeout = SecurityAnalyticsSettings.INDEX_TIMEOUT.get(this.settings);
@@ -166,15 +170,17 @@
 
         private void onGetResponse(CustomLogType logType) {
             if (logType.getSource().equals("Sigma")) {
-                onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted because source is sigma", logType.getId()), RestStatus.BAD_REQUEST));
+                onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(),
+                        "Log Type with id %s cannot be deleted because source is sigma", logType.getId()), RestStatus.BAD_REQUEST));
             }
 
             if (detectorIndices.detectorIndexExists()) {
                 searchDetectors(logType.getName(), new ActionListener<>() {
                     @Override
                     public void onResponse(SearchResponse response) {
                         if (response.isTimedOut()) {
-                            onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Search request timed out. Log Type with id %s cannot be deleted", logType.getId()), RestStatus.REQUEST_TIMEOUT));
+                            onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(),
+                                    "Search request timed out. Log Type with id %s cannot be deleted", logType.getId()), RestStatus.REQUEST_TIMEOUT));
                             return;
                         }
 
@@ -183,44 +189,7 @@
                             return;
                         }
 
-                        searchRules(logType.getName(), new ActionListener<>() {
-                            @Override
-                            public void onResponse(SearchResponse response) {
-                                if (response.isTimedOut()) {
-                                    onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Search request timed out. Log Type with id %s cannot be deleted", logType.getId()), RestStatus.REQUEST_TIMEOUT));
-                                    return;
-                                }
-
-                                if (response.getHits().getTotalHits().value > 0) {
-                                    onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted because active rules exist", logType.getId()), RestStatus.BAD_REQUEST));
-                                    return;
-                                }
-
-                                DeleteRequest deleteRequest = new DeleteRequest(LogTypeService.LOG_TYPE_INDEX, logType.getId())
-                                        .setRefreshPolicy(request.getRefreshPolicy())
-                                        .timeout(indexTimeout);
-
-                                client.delete(deleteRequest, new ActionListener<>() {
-                                    @Override
-                                    public void onResponse(DeleteResponse response) {
-                                        if (response.status() != RestStatus.OK) {
-                                            onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted", logType.getId()), RestStatus.INTERNAL_SERVER_ERROR));
-                                        }
-                                        onOperation(response);
-                                    }
-
-                                    @Override
-                                    public void onFailure(Exception e) {
-                                        onFailures(e);
-                                    }
-                                });
-                            }
-
-                            @Override
-                            public void onFailure(Exception e) {
-                                onFailures(e);
-                            }
-                        });
+                        checkRuleIndexAndDeleteLogType(logType);
                     }
 
                     @Override
@@ -229,25 +198,62 @@
                     }
                 });
             } else {
-                DeleteRequest deleteRequest = new DeleteRequest(LogTypeService.LOG_TYPE_INDEX, logType.getId())
-                        .setRefreshPolicy(request.getRefreshPolicy())
-                        .timeout(indexTimeout);
+                checkRuleIndexAndDeleteLogType(logType);
+            }
+        }
 
-                client.delete(deleteRequest, new ActionListener<>() {
+        void checkRuleIndexAndDeleteLogType(CustomLogType logType) {
+            if(ruleIndices.ruleIndexExists(false)) {
+                ruleIndices.searchRules(logType.getName(), new ActionListener<>() {
                     @Override
-                    public void onResponse(DeleteResponse response) {
-                        if (response.status() != RestStatus.OK) {
-                            onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted", logType.getId()), RestStatus.INTERNAL_SERVER_ERROR));
+                    public void onResponse(SearchResponse response) {
+                        if (response.isTimedOut()) {
+                            onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Search request timed out. Log Type with id %s cannot be deleted", logType.getId()), RestStatus.REQUEST_TIMEOUT));
+                            return;
                         }
-                        onOperation(response);
+
+                        if (response.getHits().getTotalHits().value > 0) {
+                            onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted because active rules exist", logType.getId()), RestStatus.BAD_REQUEST));
+                            return;
+                        }
+                        deleteLogType(logType);
                     }
 
                     @Override
                     public void onFailure(Exception e) {
-                        onFailures(e);
+                        if (e instanceof IndexNotFoundException) {
+                            // let log type deletion to go through if the rule index is missing
+                            deleteLogType(logType);
+                        } else {
+                            onFailures(e);
+                        }
                     }
                 });
+            } else {
+                log.warn("Custom rule index missing, allowing deletion of custom log type {} to go through", logType.getId());
+                deleteLogType(logType);
             }
+    }
+
+        private void deleteLogType(CustomLogType logType) {
+            DeleteRequest deleteRequest = new DeleteRequest(LogTypeService.LOG_TYPE_INDEX, logType.getId())
+                    .setRefreshPolicy(request.getRefreshPolicy())
+                    .timeout(indexTimeout);
+
+            client.delete(deleteRequest, new ActionListener<>() {
+                @Override
+                public void onResponse(DeleteResponse response) {
+                    if (response.status() != RestStatus.OK) {
+                        onFailures(new OpenSearchStatusException(String.format(Locale.getDefault(), "Log Type with id %s cannot be deleted", logType.getId()), RestStatus.INTERNAL_SERVER_ERROR));
+                    }
+                    onOperation(response);
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    onFailures(e);
+                }
+            });
         }
 
         private void searchDetectors(String logTypeName, ActionListener<SearchResponse> listener) {
@@ -267,23 +273,6 @@
             client.search(searchRequest, listener);
         }
 
-        private void searchRules(String logTypeName, ActionListener<SearchResponse> listener) {
-            QueryBuilder queryBuilder =
-                    QueryBuilders.nestedQuery("rule",
-                            QueryBuilders.boolQuery().must(
-                                    QueryBuilders.matchQuery("rule.category", logTypeName)
-                            ), ScoreMode.Avg);
-
-            SearchRequest searchRequest = new SearchRequest(Rule.CUSTOM_RULES_INDEX)
-                    .source(new SearchSourceBuilder()
-                            .seqNoAndPrimaryTerm(true)
-                            .version(true)
-                            .query(queryBuilder)
-                            .size(0));
-
-            client.search(searchRequest, listener);
-        }
-
         private void onOperation(DeleteResponse response) {
             this.response.set(response);
             if (counter.compareAndSet(false, true)) {
@@ -292,7 +281,7 @@
         }
 
         private void onFailures(Exception t) {
-            log.error(String.format(Locale.ROOT, "Failed to delete detector"));
+            log.error(String.format(Locale.ROOT, "Failed to delete log type"), t);
             if (counter.compareAndSet(false, true)) {
                 finishHim(null, t);
             }

src/main/java/org/opensearch/securityanalytics/transport/TransportDeleteRuleAction.java

@@ -227,6 +227,7 @@
             new DeleteByQueryRequestBuilder(client, DeleteByQueryAction.INSTANCE)
                 .source(Rule.CUSTOM_RULES_INDEX)
                 .filter(QueryBuilders.matchQuery("_id", ruleId))
+                .refresh(true)
                 .execute(new ActionListener<>() {
                     @Override
                     public void onResponse(BulkByScrollResponse response) {