diff --git a/docs/reference/indices/index-templates.asciidoc b/docs/reference/indices/index-templates.asciidoc index ecf3c88fc4c73..e50e1b3d8911a 100644 --- a/docs/reference/indices/index-templates.asciidoc +++ b/docs/reference/indices/index-templates.asciidoc @@ -96,7 +96,7 @@ PUT _index_template/template_1 [source,console] -------------------------------------------------- -DELETE _index_template/template_* +DELETE _index_template/* DELETE _component_template/* -------------------------------------------------- // TEARDOWN @@ -291,6 +291,96 @@ PUT /_index_template/template_1 In this case, an index matching `t*` will have three primary shards. If the order of composed templates were reversed, the index would have two primary shards. + +[[simulating-templates]] +===== Simulating template composition + +Since templates can be composed not only of multiple component templates, but also the index +template itself, there are two simulation APIs to determine what the resulting index settings will +be. + +To simulate the settings that would be applied to a matching index name: + +[source,console] +-------------------------------------------------- +POST /_index_template/_simulate_index/myindex +-------------------------------------------------- + +To simulate the settings that would be applied from a particular template: + +[source,console] +-------------------------------------------------- +POST /_index_template/_simulate/template_1 + +POST /_index_template/_simulate +{ + "index_patterns": ["foo"], + "template": { + "settings": { + "number_of_replicas": 0 + } + } +} +-------------------------------------------------- + + +Here's an example demonstrating simulating both an index name and template name: + +[source,console] +-------------------------------------------------- +PUT /_component_template/ct1 <1> +{ + "template": { + "settings": { + "index.number_of_shards": 2 + } + } +} + +PUT /_component_template/ct2 <2> +{ + "template": { + "settings": { + "index.number_of_replicas": 0 + }, + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + } + } + } + } +} + +PUT /_index_template/final-template <3> +{ + "index_patterns": ["logdata-*"], + "composed_of": ["ct1", "ct2"], + "priority": 5 +} + +POST /_index_template/_simulate_index/logdata-2019-02-01 <4> + +POST /_index_template/_simulate/final-template <5> + +POST /_index_template/_simulate <6> +{ + "index_patterns": ["mydata-*"], + "composed_of": ["ct2"], + "priority": 10 +} +-------------------------------------------------- +<1> Creating a component template (ct1) setting the number of shards to two +<2> Creating another component template (ct2) setting the number of replicas to zero with mappings +<3> Creating an index template called "final" template using ct1 and ct2 +<4> Simulate the settings that would be applied for a new index "logdata-2019-02-01" +<5> Simulate the settings composed using the "final-template" index template +<6> Simulate the settings composed using a custom specified template + +When simulating a template and specifying a template in the body of the request, the simulated +template is not added to the existing templates, it is only used for the simulation. + ===== Index template with index aliases You can include <> in an index template. diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.simulate_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.simulate_template.json new file mode 100644 index 0000000000000..2511ca637e6c1 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.simulate_template.json @@ -0,0 +1,51 @@ +{ + "indices.simulate_template":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-templates.html", + "description": "Simulate resolving the given template name or body" + }, + "stability":"stable", + "url":{ + "paths":[ + { + "path":"/_index_template/_simulate", + "methods":[ + "POST" + ] + }, + { + "path":"/_index_template/_simulate/{name}", + "methods":[ + "POST" + ], + "parts":{ + "name":{ + "type":"string", + "description":"The name of the index template" + } + } + } + ] + }, + "params":{ + "create":{ + "type":"boolean", + "description":"Whether the index template we optionally defined in the body should only be dry-run added if new or can also replace an existing one", + "default":false + }, + "cause":{ + "type":"string", + "description":"User defined reason for dry-run creating the new template for simulation purposes", + "default":false + }, + "master_timeout":{ + "type":"time", + "description":"Specify timeout for connection to master" + } + }, + "body":{ + "description":"New index template definition to be simulated, if no index template name is specified", + "required":false + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml index 0a03a5bd926fb..50aeee54a33f2 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml @@ -25,7 +25,7 @@ - match: {template.settings.index.number_of_shards: "1"} - match: {template.settings.index.number_of_replicas: "0"} - - match: {template.mappings._doc.properties.field.type: "keyword"} + - match: {template.mappings.properties.field.type: "keyword"} - match: {overlapping: []} --- @@ -77,7 +77,7 @@ - match: {template.settings.index.blocks.write: "true"} - match: {template.settings.index.number_of_replicas: "2"} - - match: {template.mappings._doc.properties.ct_field.type: "keyword"} + - match: {template.mappings.properties.ct_field.type: "keyword"} - match: {overlapping.0.name: existing_test} - match: {overlapping.0.index_patterns: ["te*"]} - length: {template.aliases: 1} @@ -170,7 +170,7 @@ - match: {template.settings.index.number_of_shards: "1"} - match: {template.settings.index.number_of_replicas: "0"} - - match: {template.mappings._doc.properties.field.type: "keyword"} + - match: {template.mappings.properties.field.type: "keyword"} - match: {overlapping.0.name: v1_template} - match: {overlapping.0.index_patterns: ["t*", "t1*"]} - match: {overlapping.1.name: v2_template} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.simulate_template/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.simulate_template/10_basic.yml new file mode 100644 index 0000000000000..552b6ff972b0f --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.simulate_template/10_basic.yml @@ -0,0 +1,146 @@ +--- +"Simulate template without a template in the body": + - skip: + version: " - 7.99.99" + reason: "not yet backported" + features: ["default_shards"] + + - do: + indices.put_index_template: + name: my-template + body: + index_patterns: other + template: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + field: + type: keyword + + - do: + indices.simulate_template: + name: my-template + + - match: {template.settings.index.number_of_shards: "1"} + - match: {template.settings.index.number_of_replicas: "0"} + - match: {template.mappings.properties.field.type: "keyword"} + - match: {overlapping: []} + +--- +"Simulate index template specifying a new template": + - skip: + version: " - 7.99.99" + reason: "not yet backported" + features: ["default_shards"] + + - do: + indices.put_index_template: + name: existing_test + body: + index_patterns: te* + priority: 10 + template: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + field: + type: keyword + + - do: + cluster.put_component_template: + name: ct + body: + template: + settings: + index.number_of_replicas: 2 + mappings: + properties: + ct_field: + type: keyword + + - do: + indices.simulate_template: + body: + index_patterns: te* + priority: 15 + template: + settings: + index.blocks.write: true + aliases: + test_alias: {} + composed_of: ["ct"] + + - match: {template.settings.index.blocks.write: "true"} + - match: {template.settings.index.number_of_replicas: "2"} + - match: {template.mappings.properties.ct_field.type: "keyword"} + - match: {overlapping.0.name: existing_test} + - match: {overlapping.0.index_patterns: ["te*"]} + - length: {template.aliases: 1} + - is_true: template.aliases.test_alias + +--- +"Simulate template matches overlapping V1 and V2 templates": + - skip: + version: " - 7.99.99" + reason: "not yet backported" + features: ["allowed_warnings", "default_shards"] + + - do: + indices.put_template: + name: v1_template + body: + index_patterns: [t*, t1*] + settings: + number_of_shards: 5 + + - do: + allowed_warnings: + - "index template [v2_template] has index patterns [te*] matching patterns from existing older templates [v1_template] with patterns + (v1_template => [t*, t1*]); this template [v2_template] will take precedence during new index creation" + indices.put_index_template: + name: v2_template + body: + index_patterns: te* + priority: 10 + template: + settings: + number_of_shards: 10 + number_of_replicas: 2 + mappings: + properties: + field: + type: text + + - do: + allowed_warnings: + - "index template [winning_v2_template] has index patterns [te*] matching patterns from existing older templates [v1_template] with patterns + (v1_template => [t*, t1*]); this template [winning_v2_template] will take precedence during new index creation" + indices.put_index_template: + name: winning_v2_template + body: + index_patterns: te* + priority: 20 + template: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + field: + type: keyword + + - do: + indices.simulate_template: + name: winning_v2_template + + - match: {template.settings.index.number_of_shards: "1"} + - match: {template.settings.index.number_of_replicas: "0"} + - match: {template.mappings.properties.field.type: "keyword"} + - match: {overlapping.0.name: v1_template} + - match: {overlapping.0.index_patterns: ["t*", "t1*"]} + - match: {overlapping.1.name: v2_template} + - match: {overlapping.1.index_patterns: ["te*"]} diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java index 11e7cf824942d..03687cbc56e62 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionModule.java +++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java @@ -160,7 +160,9 @@ import org.elasticsearch.action.admin.indices.template.get.TransportGetIndexTemplateV2Action; import org.elasticsearch.action.admin.indices.template.get.TransportGetIndexTemplatesAction; import org.elasticsearch.action.admin.indices.template.post.SimulateIndexTemplateAction; +import org.elasticsearch.action.admin.indices.template.post.SimulateTemplateAction; import org.elasticsearch.action.admin.indices.template.post.TransportSimulateIndexTemplateAction; +import org.elasticsearch.action.admin.indices.template.post.TransportSimulateTemplateAction; import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateAction; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateV2Action; @@ -313,7 +315,6 @@ import org.elasticsearch.rest.action.admin.indices.RestIndicesShardStoresAction; import org.elasticsearch.rest.action.admin.indices.RestIndicesStatsAction; import org.elasticsearch.rest.action.admin.indices.RestOpenIndexAction; -import org.elasticsearch.rest.action.admin.indices.RestSimulateIndexTemplateAction; import org.elasticsearch.rest.action.admin.indices.RestPutComponentTemplateAction; import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction; import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateV2Action; @@ -322,6 +323,8 @@ import org.elasticsearch.rest.action.admin.indices.RestRefreshAction; import org.elasticsearch.rest.action.admin.indices.RestResizeHandler; import org.elasticsearch.rest.action.admin.indices.RestRolloverIndexAction; +import org.elasticsearch.rest.action.admin.indices.RestSimulateIndexTemplateAction; +import org.elasticsearch.rest.action.admin.indices.RestSimulateTemplateAction; import org.elasticsearch.rest.action.admin.indices.RestSyncedFlushAction; import org.elasticsearch.rest.action.admin.indices.RestUpdateSettingsAction; import org.elasticsearch.rest.action.admin.indices.RestUpgradeActionDeprecated; @@ -540,6 +543,7 @@ public void reg actions.register(GetIndexTemplateV2Action.INSTANCE, TransportGetIndexTemplateV2Action.class); actions.register(DeleteIndexTemplateV2Action.INSTANCE, TransportDeleteIndexTemplateV2Action.class); actions.register(SimulateIndexTemplateAction.INSTANCE, TransportSimulateIndexTemplateAction.class); + actions.register(SimulateTemplateAction.INSTANCE, TransportSimulateTemplateAction.class); actions.register(ValidateQueryAction.INSTANCE, TransportValidateQueryAction.class); actions.register(RefreshAction.INSTANCE, TransportRefreshAction.class); actions.register(FlushAction.INSTANCE, TransportFlushAction.class); @@ -690,6 +694,7 @@ public void initRestHandlers(Supplier nodesInCluster) { registerHandler.accept(new RestGetIndexTemplateV2Action()); registerHandler.accept(new RestDeleteIndexTemplateV2Action()); registerHandler.accept(new RestSimulateIndexTemplateAction()); + registerHandler.accept(new RestSimulateTemplateAction()); registerHandler.accept(new RestPutMappingAction()); registerHandler.accept(new RestGetMappingAction()); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateAction.java new file mode 100644 index 0000000000000..1c6ccc467c345 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateAction.java @@ -0,0 +1,136 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.template.post; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.ValidateActions; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateV2Action; +import org.elasticsearch.action.support.master.MasterNodeReadRequest; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.util.Objects; + +/** + * An action for simulating the complete composed settings of the specified + * index template name, or index template configuration + */ +public class SimulateTemplateAction extends ActionType { + + public static final SimulateTemplateAction INSTANCE = new SimulateTemplateAction(); + public static final String NAME = "indices:admin/index_template/simulate"; + + private SimulateTemplateAction() { + super(NAME, SimulateIndexTemplateResponse::new); + } + + public static class Request extends MasterNodeReadRequest { + + @Nullable + private String templateName; + + @Nullable + private PutIndexTemplateV2Action.Request indexTemplateRequest; + + public Request() { } + + public Request(String templateName) { + if (templateName == null) { + throw new IllegalArgumentException("template name cannot be null"); + } + this.templateName = templateName; + } + + public Request(PutIndexTemplateV2Action.Request indexTemplateRequest) { + if (indexTemplateRequest == null) { + throw new IllegalArgumentException("index template body must be present"); + } + this.indexTemplateRequest = indexTemplateRequest; + } + + public Request(StreamInput in) throws IOException { + super(in); + templateName = in.readOptionalString(); + indexTemplateRequest = in.readOptionalWriteable(PutIndexTemplateV2Action.Request::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeOptionalString(templateName); + out.writeOptionalWriteable(indexTemplateRequest); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (indexTemplateRequest != null) { + validationException = indexTemplateRequest.validateIndexTemplate(validationException); + } + if (templateName == null && indexTemplateRequest == null) { + validationException = + ValidateActions.addValidationError("either index name or index template body must be specified for simulation", + validationException); + } + return validationException; + } + + @Nullable + public String getTemplateName() { + return templateName; + } + + @Nullable + public PutIndexTemplateV2Action.Request getIndexTemplateRequest() { + return indexTemplateRequest; + } + + public Request templateName(String templateName) { + this.templateName = templateName; + return this; + } + + public Request indexTemplateRequest(PutIndexTemplateV2Action.Request indexTemplateRequest) { + this.indexTemplateRequest = indexTemplateRequest; + return this; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Request that = (Request) o; + return templateName.equals(that.templateName) && + Objects.equals(indexTemplateRequest, that.indexTemplateRequest); + } + + @Override + public int hashCode() { + return Objects.hash(templateName, indexTemplateRequest); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index 0b65bba65b979..964679771e2b0 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -44,7 +44,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -99,30 +98,51 @@ protected SimulateIndexTemplateResponse read(StreamInput in) throws IOException @Override protected void masterOperation(Task task, SimulateIndexTemplateRequest request, ClusterState state, ActionListener listener) throws Exception { - ClusterState simulateOnClusterState = state; + final ClusterState stateWithTemplate; if (request.getIndexTemplateRequest() != null) { // we'll "locally" add the template defined by the user in the cluster state (as if it existed in the system) - String simulateTemplateToAdd = "simulate_new_template_" + UUIDs.randomBase64UUID().toLowerCase(Locale.ROOT); - simulateOnClusterState = indexTemplateService.addIndexTemplateV2(state, request.getIndexTemplateRequest().create(), + String simulateTemplateToAdd = "simulate_index_template_" + UUIDs.randomBase64UUID().toLowerCase(Locale.ROOT); + // Perform validation for things like typos in component template names + MetadataIndexTemplateService.validateV2TemplateRequest(state.metadata(), simulateTemplateToAdd, + request.getIndexTemplateRequest().indexTemplate()); + stateWithTemplate = indexTemplateService.addIndexTemplateV2(state, request.getIndexTemplateRequest().create(), simulateTemplateToAdd, request.getIndexTemplateRequest().indexTemplate()); + } else { + stateWithTemplate = state; } - String matchingTemplate = findV2Template(simulateOnClusterState.metadata(), request.getIndexName(), false); + String matchingTemplate = findV2Template(stateWithTemplate.metadata(), request.getIndexName(), false); if (matchingTemplate == null) { listener.onResponse(new SimulateIndexTemplateResponse(null, null)); return; } - Settings settings = resolveSettings(simulateOnClusterState.metadata(), matchingTemplate); - // empty request mapping as the user can't specify any explicit mappings via the simulate api - Map mappings = resolveV2Mappings("{}", simulateOnClusterState, matchingTemplate, xContentRegistry); - String mappingsJson = Strings.toString(XContentFactory.jsonBuilder() - .startObject() - .field(MapperService.SINGLE_MAPPING_NAME, mappings) - .endObject()); + final ClusterState tempClusterState = resolveTemporaryState(matchingTemplate, request.getIndexName(), stateWithTemplate); + IndexTemplateV2 templateV2 = tempClusterState.metadata().templatesV2().get(matchingTemplate); + assert templateV2 != null : "the matched template must exist"; - List> resolvedAliases = MetadataIndexTemplateService.resolveAliases(simulateOnClusterState.metadata(), - matchingTemplate); + final Template template = resolveTemplate(matchingTemplate, request.getIndexName(), stateWithTemplate, + xContentRegistry, indicesService, aliasValidator); + + final Map> overlapping = new HashMap<>(); + overlapping.putAll(findConflictingV1Templates(tempClusterState, matchingTemplate, templateV2.indexPatterns())); + overlapping.putAll(findConflictingV2Templates(tempClusterState, matchingTemplate, templateV2.indexPatterns())); + + listener.onResponse(new SimulateIndexTemplateResponse(template, overlapping)); + } + + @Override + protected ClusterBlockException checkBlock(SimulateIndexTemplateRequest request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); + } + + /** + * Return a temporary cluster state with an index that exists using the + * matched template's settings + */ + public static ClusterState resolveTemporaryState(final String matchingTemplate, final String indexName, + final ClusterState simulatedState) { + Settings settings = resolveSettings(simulatedState.metadata(), matchingTemplate); // create the index with dummy settings in the cluster state so we can parse and validate the aliases Settings dummySettings = Settings.builder() @@ -132,34 +152,56 @@ protected void masterOperation(Task task, SimulateIndexTemplateRequest request, .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()) .build(); - final IndexMetadata indexMetadata = IndexMetadata.builder(request.getIndexName()).settings(dummySettings).build(); + final IndexMetadata indexMetadata = IndexMetadata.builder(indexName).settings(dummySettings).build(); - final ClusterState tempClusterState = ClusterState.builder(simulateOnClusterState) - .metadata(Metadata.builder(simulateOnClusterState.metadata()) - .put(indexMetadata, true) - .build()) - .build(); - List aliases = indicesService.withTempIndexService(indexMetadata, tempIndexService -> - MetadataCreateIndexService.resolveAndValidateAliases(request.getIndexName(), Set.of(), - resolvedAliases, tempClusterState.metadata(), aliasValidator, xContentRegistry, - // the context is only used for validation so it's fine to pass fake values for the - // shard id and the current timestamp - tempIndexService.newQueryShardContext(0, null, () -> 0L, null))); + return ClusterState.builder(simulatedState) + .metadata(Metadata.builder(simulatedState.metadata()) + .put(indexMetadata, true) + .build()) + .build(); + } - IndexTemplateV2 templateV2 = tempClusterState.metadata().templatesV2().get(matchingTemplate); - assert templateV2 != null : "the matched template must exist"; + /** + * Take a template and index name as well as state where the template exists, and return a final + * {@link Template} that represents all the resolved Settings, Mappings, and Aliases + */ + public static Template resolveTemplate(final String matchingTemplate, final String indexName, + final ClusterState simulatedState, + final NamedXContentRegistry xContentRegistry, + final IndicesService indicesService, + final AliasValidator aliasValidator) throws Exception { + Settings settings = resolveSettings(simulatedState.metadata(), matchingTemplate); - Map> overlapping = new HashMap<>(); - overlapping.putAll(findConflictingV1Templates(tempClusterState, matchingTemplate, templateV2.indexPatterns())); - overlapping.putAll(findConflictingV2Templates(tempClusterState, matchingTemplate, templateV2.indexPatterns())); + // empty request mapping as the user can't specify any explicit mappings via the simulate api + Map mappings = resolveV2Mappings("{}", simulatedState, matchingTemplate, xContentRegistry); + String mappingsJson = Strings.toString(XContentFactory.jsonBuilder().map(mappings)); - Template template = new Template(settings, mappingsJson == null ? null : new CompressedXContent(mappingsJson), - aliases.stream().collect(Collectors.toMap(AliasMetadata::getAlias, Function.identity()))); - listener.onResponse(new SimulateIndexTemplateResponse(template, overlapping)); - } + List> resolvedAliases = MetadataIndexTemplateService.resolveAliases(simulatedState.metadata(), + matchingTemplate); - @Override - protected ClusterBlockException checkBlock(SimulateIndexTemplateRequest request, ClusterState state) { - return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); + // create the index with dummy settings in the cluster state so we can parse and validate the aliases + Settings dummySettings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(settings) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()) + .build(); + final IndexMetadata indexMetadata = IndexMetadata.builder(indexName).settings(dummySettings).build(); + + final ClusterState tempClusterState = ClusterState.builder(simulatedState) + .metadata(Metadata.builder(simulatedState.metadata()) + .put(indexMetadata, true) + .build()) + .build(); + List aliases = indicesService.withTempIndexService(indexMetadata, tempIndexService -> + MetadataCreateIndexService.resolveAndValidateAliases(indexName, Set.of(), + resolvedAliases, tempClusterState.metadata(), aliasValidator, xContentRegistry, + // the context is only used for validation so it's fine to pass fake values for the + // shard id and the current timestamp + tempIndexService.newQueryShardContext(0, null, () -> 0L, null))); + + return new Template(settings, mappingsJson == null ? null : new CompressedXContent(mappingsJson), + aliases.stream().collect(Collectors.toMap(AliasMetadata::getAlias, Function.identity()))); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java new file mode 100644 index 0000000000000..38f70be42e612 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java @@ -0,0 +1,149 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.template.post; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.AliasValidator; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.IndexTemplateV2; +import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findConflictingV1Templates; +import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findConflictingV2Templates; + +/** + * Handles simulating an index template either by name (looking it up in the + * cluster state), or by a provided template configuration + */ +public class TransportSimulateTemplateAction + extends TransportMasterNodeReadAction { + + private final MetadataIndexTemplateService indexTemplateService; + private final NamedXContentRegistry xContentRegistry; + private final IndicesService indicesService; + private AliasValidator aliasValidator; + + @Inject + public TransportSimulateTemplateAction(TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, MetadataIndexTemplateService indexTemplateService, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + NamedXContentRegistry xContentRegistry, IndicesService indicesService) { + super(SimulateTemplateAction.NAME, transportService, clusterService, threadPool, actionFilters, + SimulateTemplateAction.Request::new, indexNameExpressionResolver); + this.indexTemplateService = indexTemplateService; + this.xContentRegistry = xContentRegistry; + this.indicesService = indicesService; + this.aliasValidator = new AliasValidator(); + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected SimulateIndexTemplateResponse read(StreamInput in) throws IOException { + return new SimulateIndexTemplateResponse(in); + } + + @Override + protected void masterOperation(Task task, SimulateTemplateAction.Request request, ClusterState state, + ActionListener listener) throws Exception { + String uuid = UUIDs.randomBase64UUID().toLowerCase(Locale.ROOT); + final String temporaryIndexName = "simulate_template_index_" + uuid; + final ClusterState stateWithTemplate; + final String simulateTemplateToAdd; + + // First, if a template body was requested, we need to "fake add" that template to the + // cluster state, so it can be used when we resolved settings/etc + if (request.getIndexTemplateRequest() != null) { + // we'll "locally" add the template defined by the user in the cluster state (as if it existed in the system) + simulateTemplateToAdd = "simulate_template_" + uuid; + // Perform validation for things like typos in component template names + MetadataIndexTemplateService.validateV2TemplateRequest(state.metadata(), simulateTemplateToAdd, + request.getIndexTemplateRequest().indexTemplate()); + stateWithTemplate = indexTemplateService.addIndexTemplateV2(state, request.getIndexTemplateRequest().create(), + simulateTemplateToAdd, request.getIndexTemplateRequest().indexTemplate()); + } else { + simulateTemplateToAdd = null; + stateWithTemplate = state; + } + + // We also need the name of the template we're going to resolve, so if they specified a + // name, use that, otherwise use the name of the template that was "fake added" in the previous block + final String matchingTemplate; + if (request.getTemplateName() == null) { + // Automatically match the template that was added + matchingTemplate = simulateTemplateToAdd; + } else { + matchingTemplate = request.getTemplateName(); + } + + // If they didn't either specify a name that existed or a template body, we cannot simulate anything! + if (matchingTemplate == null) { + // They should have specified either a template name or the body of a template, but neither were specified + listener.onFailure(new IllegalArgumentException("unable to find a matching template")); + return; + } else if (stateWithTemplate.metadata().templatesV2().containsKey(matchingTemplate) == false) { + // They specified a template, but it didn't exist + listener.onFailure(new IllegalArgumentException("unable to simulate template [" + matchingTemplate + "] that does not exist")); + return; + } + + final ClusterState tempClusterState = + TransportSimulateIndexTemplateAction.resolveTemporaryState(matchingTemplate, temporaryIndexName, stateWithTemplate); + IndexTemplateV2 templateV2 = tempClusterState.metadata().templatesV2().get(matchingTemplate); + assert templateV2 != null : "the matched template must exist"; + + Map> overlapping = new HashMap<>(); + overlapping.putAll(findConflictingV1Templates(tempClusterState, matchingTemplate, templateV2.indexPatterns())); + overlapping.putAll(findConflictingV2Templates(tempClusterState, matchingTemplate, templateV2.indexPatterns())); + + Template template = TransportSimulateIndexTemplateAction.resolveTemplate(matchingTemplate, temporaryIndexName, + stateWithTemplate, xContentRegistry, indicesService, aliasValidator); + listener.onResponse(new SimulateIndexTemplateResponse(template, overlapping)); + } + + @Override + protected ClusterBlockException checkBlock(SimulateTemplateAction.Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); + } +} diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index a404fc386331a..428bb31138c7f 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -352,7 +352,7 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS }); } - static void validateV2TemplateRequest(Metadata metadata, String name, IndexTemplateV2 template) { + public static void validateV2TemplateRequest(Metadata metadata, String name, IndexTemplateV2 template) { if (template.indexPatterns().stream().anyMatch(Regex::isMatchAllPattern)) { Settings mergedSettings = resolveSettings(metadata, template); if (IndexMetadata.INDEX_HIDDEN_SETTING.exists(mergedSettings)) { diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateTemplateAction.java new file mode 100644 index 0000000000000..7ff2518b82d37 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateTemplateAction.java @@ -0,0 +1,64 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.rest.action.admin.indices; + +import org.elasticsearch.action.admin.indices.template.post.SimulateTemplateAction; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateV2Action; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.cluster.metadata.IndexTemplateV2; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.POST; + +public class RestSimulateTemplateAction extends BaseRestHandler { + @Override + public List routes() { + return List.of( + new Route(POST, "/_index_template/_simulate"), + new Route(POST, "/_index_template/_simulate/{name}")); + } + + @Override + public String getName() { + return "simulate_template_action"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + SimulateTemplateAction.Request simulateRequest = new SimulateTemplateAction.Request(); + simulateRequest.templateName(request.param("name")); + if (request.hasContent()) { + PutIndexTemplateV2Action.Request indexTemplateRequest = new PutIndexTemplateV2Action.Request("simulating_template"); + indexTemplateRequest.indexTemplate(IndexTemplateV2.parse(request.contentParser())); + indexTemplateRequest.create(request.paramAsBoolean("create", false)); + indexTemplateRequest.cause(request.param("cause", "api")); + + simulateRequest.indexTemplateRequest(indexTemplateRequest); + } + simulateRequest.masterNodeTimeout(request.paramAsTime("master_timeout", simulateRequest.masterNodeTimeout())); + + return channel -> client.execute(SimulateTemplateAction.INSTANCE, simulateRequest, new RestToXContentListener<>(channel)); + } +} diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java new file mode 100644 index 0000000000000..c0804952bacd8 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java @@ -0,0 +1,81 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.template.post; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateV2Action; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.IndexTemplateV2; +import org.elasticsearch.cluster.metadata.IndexTemplateV2Tests; +import org.elasticsearch.cluster.metadata.Template; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.AbstractWireSerializingTestCase; + +import java.io.IOException; +import java.util.List; + +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; + +public class SimulateTemplateRequestTests extends AbstractWireSerializingTestCase { + + @Override + protected Writeable.Reader instanceReader() { + return SimulateTemplateAction.Request::new; + } + + @Override + protected SimulateTemplateAction.Request createTestInstance() { + SimulateTemplateAction.Request req = new SimulateTemplateAction.Request(randomAlphaOfLength(10)); + PutIndexTemplateV2Action.Request newTemplateRequest = new PutIndexTemplateV2Action.Request(randomAlphaOfLength(4)); + newTemplateRequest.indexTemplate(IndexTemplateV2Tests.randomInstance()); + req.indexTemplateRequest(newTemplateRequest); + return req; + } + + @Override + protected SimulateTemplateAction.Request mutateInstance(SimulateTemplateAction.Request instance) throws IOException { + return randomValueOtherThan(instance, this::createTestInstance); + } + + public void testIndexNameCannotBeNullOrEmpty() { + expectThrows(IllegalArgumentException.class, () -> new SimulateTemplateAction.Request((String) null)); + expectThrows(IllegalArgumentException.class, () -> new SimulateTemplateAction.Request((PutIndexTemplateV2Action.Request) null)); + } + + public void testAddingGlobalTemplateWithHiddenIndexSettingIsIllegal() { + Template template = new Template(Settings.builder().put(IndexMetadata.SETTING_INDEX_HIDDEN, true).build(), null, null); + IndexTemplateV2 globalTemplate = new IndexTemplateV2(List.of("*"), template, null, null, null, null, null); + + PutIndexTemplateV2Action.Request request = new PutIndexTemplateV2Action.Request("test"); + request.indexTemplate(globalTemplate); + + SimulateTemplateAction.Request simulateRequest = new SimulateTemplateAction.Request("testing"); + simulateRequest.indexTemplateRequest(request); + + ActionRequestValidationException validationException = simulateRequest.validate(); + assertThat(validationException, is(notNullValue())); + List validationErrors = validationException.validationErrors(); + assertThat(validationErrors.size(), is(1)); + String error = validationErrors.get(0); + assertThat(error, is("global V2 templates may not specify the setting " + IndexMetadata.SETTING_INDEX_HIDDEN)); + } +}