Support specifying multiple templates names in delete composable inde…

…x template api (#70298)

Backporting #70094 to 7.x branch.

Add support to delete composable index templates api to specify multiple template
names separated by a comma.

Change to cleanup template logic for rest tests to remove all composable index templates via a single delete composable index template request. This to optimize the cleanup logic. After each rest test we delete all templates. So deleting templates this via a single api call (and thus single cluster state update) saves a lot of time considering the number of rest tests.

Relates to #69973

docs/reference/indices/delete-index-template.asciidoc

@@ -33,6 +33,10 @@ DELETE /_index_template/my-index-template
 
 `DELETE /_index_template/<index-template>`
 
+The provided <index-template> may contain multiple template names separated by a comma.
+If multiple template names are specified then there is no wildcard support and the
+provided names should match completely with existing templates.
+
 [[delete-template-api-prereqs]]
 ==== {api-prereq-title}
 
@@ -44,8 +48,8 @@ privilege>> to use this API.
 ==== {api-description-title}
 
 Use the delete index template API to delete one or more index templates.
-Index templates define <<index-modules-settings,settings>>, <<mapping,mappings>>, 
-and <<indices-aliases,aliases>> that can be applied automatically to new indices. 
+Index templates define <<index-modules-settings,settings>>, <<mapping,mappings>>,
+and <<indices-aliases,aliases>> that can be applied automatically to new indices.
 
 
 [[delete-template-api-path-params]]

server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/DeleteComposableIndexTemplateAction.java

@@ -8,14 +8,17 @@
 
 package org.elasticsearch.action.admin.indices.template.delete;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.action.support.master.MasterNodeRequest;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Objects;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
@@ -31,34 +34,28 @@ private DeleteComposableIndexTemplateAction() {
 
     public static class Request extends MasterNodeRequest<Request> {
 
-        private String name;
+        private final String[] names;
 
         public Request(StreamInput in) throws IOException {
             super(in);
-            name = in.readString();
+            if (in.getVersion().onOrAfter(Version.V_7_13_0)) {
+                names = in.readStringArray();
+            } else {
+                names = new String[] {in.readString()};
+            }
         }
 
-        public Request() { }
-
         /**
          * Constructs a new delete template request for the specified name.
          */
-        public Request(String name) {
-            this.name = name;
-        }
-
-        /**
-         * Set the index template name to delete.
-         */
-        public Request name(String name) {
-            this.name = name;
-            return this;
+        public Request(String... names) {
+            this.names = Objects.requireNonNull(names, "templates to delete must not be null");
         }
 
         @Override
         public ActionRequestValidationException validate() {
             ActionRequestValidationException validationException = null;
-            if (name == null) {
+            if (Arrays.stream(names).anyMatch(Strings::hasLength) == false) {
                 validationException = addValidationError("name is missing", validationException);
             }
             return validationException;
@@ -67,19 +64,23 @@ public ActionRequestValidationException validate() {
         /**
          * The index template name to delete.
          */
-        public String name() {
-            return name;
+        public String[] names() {
+            return names;
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
-            out.writeString(name);
+            if (out.getVersion().onOrAfter(Version.V_7_13_0)) {
+                out.writeStringArray(names);
+            } else {
+                out.writeString(names[0]);
+            }
         }
 
         @Override
         public int hashCode() {
-            return name.hashCode();
+            return Arrays.hashCode(names);
         }
 
         @Override
@@ -91,7 +92,7 @@ public boolean equals(Object obj) {
                 return false;
             }
             Request other = (Request) obj;
-            return Objects.equals(other.name, this.name);
+            return Arrays.equals(other.names, this.names);
         }
     }
 }

server/src/main/java/org/elasticsearch/action/admin/indices/template/delete/TransportDeleteComposableIndexTemplateAction.java

@@ -45,6 +45,6 @@ protected ClusterBlockException checkBlock(DeleteComposableIndexTemplateAction.R
     @Override
     protected void masterOperation(final DeleteComposableIndexTemplateAction.Request request, final ClusterState state,
                                    final ActionListener<AcknowledgedResponse> listener) {
-        indexTemplateService.removeIndexTemplateV2(request.name(), request.masterNodeTimeout(), listener);
+        indexTemplateService.removeIndexTemplateV2(request.names(), request.masterNodeTimeout(), listener);
     }
 }

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java

@@ -57,6 +57,7 @@
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Locale;
@@ -609,9 +610,9 @@ static Map<String, List<String>> findConflictingV2Templates(final ClusterState s
      * Remove the given index template from the cluster state. The index template name
      * supports simple regex wildcards for removing multiple index templates at a time.
      */
-    public void removeIndexTemplateV2(final String name, final TimeValue masterTimeout,
+    public void removeIndexTemplateV2(final String[] names, final TimeValue masterTimeout,
                                       final ActionListener<AcknowledgedResponse> listener) {
-        clusterService.submitStateUpdateTask("remove-index-template-v2 [" + name + "]",
+        clusterService.submitStateUpdateTask("remove-index-template-v2 [" + String.join(",", names) + "]",
             new ClusterStateUpdateTask(Priority.URGENT, masterTimeout) {
 
                 @Override
@@ -621,7 +622,7 @@ public void onFailure(String source, Exception e) {
 
                 @Override
                 public ClusterState execute(ClusterState currentState) {
-                    return innerRemoveIndexTemplateV2(currentState, name);
+                    return innerRemoveIndexTemplateV2(currentState, names);
                 }
 
                 @Override
@@ -632,20 +633,47 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS
     }
 
     // Package visible for testing
-    static ClusterState innerRemoveIndexTemplateV2(ClusterState currentState, String name) {
+    static ClusterState innerRemoveIndexTemplateV2(ClusterState currentState, String... names) {
         Set<String> templateNames = new HashSet<>();
-        for (String templateName : currentState.metadata().templatesV2().keySet()) {
-            if (Regex.simpleMatch(name, templateName)) {
-                templateNames.add(templateName);
+
+        if (names.length > 1) {
+            Set<String> missingNames = null;
+            for (String name : names) {
+                if (currentState.metadata().templatesV2().containsKey(name)) {
+                    templateNames.add(name);
+                } else {
+                    // wildcards are not supported, so if a name with a wildcard is specified then
+                    // the else clause gets executed, because template names can't contain a wildcard.
+                    if (missingNames == null) {
+                        missingNames = new LinkedHashSet<>();
+                    }
+                    missingNames.add(name);
+                }
             }
-        }
-        if (templateNames.isEmpty()) {
-            // if its a match all pattern, and no templates are found (we have none), don't
-            // fail with index missing...
-            if (Regex.isMatchAllPattern(name)) {
-                return currentState;
+
+            if (missingNames != null) {
+                throw new IndexTemplateMissingException(String.join(",", missingNames));
+            }
+        } else {
+            final String name = names[0];
+            for (String templateName : currentState.metadata().templatesV2().keySet()) {
+                if (Regex.simpleMatch(name, templateName)) {
+                    templateNames.add(templateName);
+                }
+            }
+            if (templateNames.isEmpty()) {
+                // if its a match all pattern, and no templates are found (we have none), don't
+                // fail with index missing...
+                boolean isMatchAll = false;
+                if (Regex.isMatchAllPattern(name)) {
+                    isMatchAll = true;
+                }
+                if (isMatchAll) {
+                    return currentState;
+                } else {
+                    throw new IndexTemplateMissingException(name);
+                }
             }
-            throw new IndexTemplateMissingException(name);
         }
 
         Optional<Set<String>> dataStreamsUsingTemplates = templateNames.stream()

server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestDeleteComposableIndexTemplateAction.java

@@ -10,6 +10,7 @@
 
 import org.elasticsearch.action.admin.indices.template.delete.DeleteComposableIndexTemplateAction;
 import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.RestToXContentListener;
@@ -35,7 +36,8 @@ public String getName() {
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
 
-        DeleteComposableIndexTemplateAction.Request deleteReq = new DeleteComposableIndexTemplateAction.Request(request.param("name"));
+        String[] names = Strings.splitStringByCommaToArray(request.param("name"));
+        DeleteComposableIndexTemplateAction.Request deleteReq = new DeleteComposableIndexTemplateAction.Request(names);
         deleteReq.masterNodeTimeout(request.paramAsTime("master_timeout", deleteReq.masterNodeTimeout()));
 
         return channel -> client.execute(DeleteComposableIndexTemplateAction.INSTANCE, deleteReq, new RestToXContentListener<>(channel));

server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java

@@ -76,6 +76,9 @@
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.matchesRegex;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.Matchers.sameInstance;
 
 public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase {
 
@@ -460,6 +463,71 @@ public void testRemoveIndexTemplateV2() throws Exception {
         assertNull(updatedState.metadata().templatesV2().get("foo"));
     }
 
+    public void testRemoveIndexTemplateV2Wildcards() throws Exception {
+        ComposableIndexTemplate template = ComposableIndexTemplateTests.randomInstance();
+        MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService();
+        ClusterState result = MetadataIndexTemplateService.innerRemoveIndexTemplateV2(ClusterState.EMPTY_STATE, "*");
+        assertThat(result, sameInstance(ClusterState.EMPTY_STATE));
+
+        ClusterState state = metadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "foo", template);
+        assertThat(state.metadata().templatesV2().get("foo"), notNullValue());
+        assertTemplatesEqual(state.metadata().templatesV2().get("foo"), template);
+
+        Exception e = expectThrows(IndexTemplateMissingException.class,
+            () -> MetadataIndexTemplateService.innerRemoveIndexTemplateV2(state, "foob*"));
+        assertThat(e.getMessage(), equalTo("index_template [foob*] missing"));
+
+        ClusterState updatedState = MetadataIndexTemplateService.innerRemoveIndexTemplateV2(state, "foo*");
+        assertThat(updatedState.metadata().templatesV2().get("foo"), nullValue());
+    }
+
+    public void testRemoveMultipleIndexTemplateV2() throws Exception {
+        ComposableIndexTemplate fooTemplate = ComposableIndexTemplateTests.randomInstance();
+        ComposableIndexTemplate barTemplate = ComposableIndexTemplateTests.randomInstance();
+        ComposableIndexTemplate bazTemplate = ComposableIndexTemplateTests.randomInstance();
+        MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService();
+
+        ClusterState state = metadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "foo", fooTemplate);
+        state = metadataIndexTemplateService.addIndexTemplateV2(state, false, "bar", barTemplate);
+        state = metadataIndexTemplateService.addIndexTemplateV2(state, false, "baz", bazTemplate);
+        assertNotNull(state.metadata().templatesV2().get("foo"));
+        assertNotNull(state.metadata().templatesV2().get("bar"));
+        assertNotNull(state.metadata().templatesV2().get("baz"));
+        assertTemplatesEqual(state.metadata().templatesV2().get("foo"), fooTemplate);
+        assertTemplatesEqual(state.metadata().templatesV2().get("bar"), barTemplate);
+        assertTemplatesEqual(state.metadata().templatesV2().get("baz"), bazTemplate);
+
+        ClusterState updatedState = MetadataIndexTemplateService.innerRemoveIndexTemplateV2(state, "foo", "baz");
+        assertNull(updatedState.metadata().templatesV2().get("foo"));
+        assertNotNull(updatedState.metadata().templatesV2().get("bar"));
+        assertNull(updatedState.metadata().templatesV2().get("baz"));
+    }
+
+    public void testRemoveMultipleIndexTemplateV2Wildcards() throws Exception {
+        ComposableIndexTemplate fooTemplate = ComposableIndexTemplateTests.randomInstance();
+        ComposableIndexTemplate barTemplate = ComposableIndexTemplateTests.randomInstance();
+        ComposableIndexTemplate bazTemplate = ComposableIndexTemplateTests.randomInstance();
+        MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService();
+
+        final ClusterState state;
+        {
+            ClusterState cs = metadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "foo", fooTemplate);
+            cs = metadataIndexTemplateService.addIndexTemplateV2(cs, false, "bar", barTemplate);
+            state = metadataIndexTemplateService.addIndexTemplateV2(cs, false, "baz", bazTemplate);
+        }
+
+        Exception e = expectThrows(IndexTemplateMissingException.class,
+            () -> MetadataIndexTemplateService.innerRemoveIndexTemplateV2(state, "foo", "b*", "k*", "*"));
+        assertThat(e.getMessage(), equalTo("index_template [b*,k*,*] missing"));
+
+        assertNotNull(state.metadata().templatesV2().get("foo"));
+        assertNotNull(state.metadata().templatesV2().get("bar"));
+        assertNotNull(state.metadata().templatesV2().get("baz"));
+        assertTemplatesEqual(state.metadata().templatesV2().get("foo"), fooTemplate);
+        assertTemplatesEqual(state.metadata().templatesV2().get("bar"), barTemplate);
+        assertTemplatesEqual(state.metadata().templatesV2().get("baz"), bazTemplate);
+    }
+
     /**
      * Test that if we have a pre-existing v1 template and put a v2 template that would match the same indices, we generate a warning
      */

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -609,29 +609,36 @@ private void wipeCluster() throws Exception {
                         EntityUtils.toString(adminClient().performRequest(getTemplatesRequest).getEntity()), false);
                     List<String> names = ((List<?>) composableIndexTemplates.get("index_templates")).stream()
                         .map(ct -> (String) ((Map<?, ?>) ct).get("name"))
+                        .filter(name -> isXPackTemplate(name) == false)
                         .collect(Collectors.toList());
-                    for (String name : names) {
-                        if (isXPackTemplate(name)) {
-                            continue;
-                        }
+                    // Ideally we would want to check the version of the elected master node and
+                    // send the delete request directly to that node.
+                    if (nodeVersions.stream().allMatch(version -> version.onOrAfter(Version.V_7_13_0))) {
                         try {
-                            adminClient().performRequest(new Request("DELETE", "_index_template/" + name));
+                            adminClient().performRequest(new Request("DELETE", "_index_template/" + String.join(",", names)));
                         } catch (ResponseException e) {
-                            logger.debug(new ParameterizedMessage("unable to remove index template {}", name), e);
+                            logger.debug(new ParameterizedMessage("unable to remove multiple composable index template {}", names), e);
+                        }
+                    } else {
+                        for (String name : names) {
+                            try {
+                                adminClient().performRequest(new Request("DELETE", "_index_template/" + name));
+                            } catch (ResponseException e) {
+                                logger.debug(new ParameterizedMessage("unable to remove composable index template {}", name), e);
+                            }
                         }
                     }
                 } catch (Exception e) {
-                    logger.info("ignoring exception removing all composable index templates", e);
+                    logger.debug("ignoring exception removing all composable index templates", e);
                     // We hit a version of ES that doesn't support index templates v2 yet, so it's safe to ignore
                 }
                 try {
                     Request compReq = new Request("GET", "_component_template");
                     compReq.setOptions(allowTypesRemovalWarnings());
                     String componentTemplates = EntityUtils.toString(adminClient().performRequest(compReq).getEntity());
                     Map<String, Object> cTemplates = XContentHelper.convertToMap(JsonXContent.jsonXContent, componentTemplates, false);
-                    @SuppressWarnings("unchecked")
-                    List<String> names = ((List<Map<String, Object>>) cTemplates.get("component_templates")).stream()
-                        .map(ct -> (String) ct.get("name"))
+                    List<String> names = ((List<?>) cTemplates.get("component_templates")).stream()
+                        .map(ct -> (String) ((Map<?, ?>) ct).get("name"))
                         .collect(Collectors.toList());
                     for (String componentTemplate : names) {
                         try {
@@ -644,7 +651,7 @@ private void wipeCluster() throws Exception {
                         }
                     }
                 } catch (Exception e) {
-                    logger.info("ignoring exception removing all component templates", e);
+                    logger.debug("ignoring exception removing all component templates", e);
                     // We hit a version of ES that doesn't support index templates v2 yet, so it's safe to ignore
                 }
                 Request getLegacyTemplatesRequest = new Request("GET", "_template");