From b63b50b945d4c44c70d56a9ce2ba9a25dd897147 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 30 Jan 2019 10:28:24 +0100 Subject: [PATCH] Give precedence to index creation when mixing typed templates with typeless index creation and vice-versa. (#37871) Currently if you mix typed templates and typeless index creation or typeless templates and typed index creation then you will end up with an error because Elasticsearch tries to create an index that has multiple types: `_doc` and the explicit type name that you used. This commit proposes to give precedence to the index creation call so that the type from the template will be ignored if the index creation call is typeless while the template is typed, and the type from the index creation call will be used if there is a typeless template. This is consistent with the fact that index creation already "wins" if a field is defined differently in the index creation call and in a template: the definition from the index creation call is used in such cases. Closes #37773 --- .../mapping/removal_of_types.asciidoc | 74 ++++++++++ .../20_mix_typeless_typeful.yml | 137 ++++++++++++++++++ .../metadata/MetaDataCreateIndexService.java | 22 +++ .../metadata/IndexCreationTaskTests.java | 26 ++++ 4 files changed, 259 insertions(+) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/20_mix_typeless_typeful.yml diff --git a/docs/reference/mapping/removal_of_types.asciidoc b/docs/reference/mapping/removal_of_types.asciidoc index ee5ee4b4fe664..b9066a4c7af49 100644 --- a/docs/reference/mapping/removal_of_types.asciidoc +++ b/docs/reference/mapping/removal_of_types.asciidoc @@ -534,3 +534,77 @@ PUT index/_doc/1 The <>, <>, <> and <> APIs will continue to return a `_type` key in the response in 7.0, but it is considered deprecated and will be removed in 8.0. + +[float] +=== Index templates + +It is recommended to make index templates typeless before upgrading to 7.0 by +re-adding them with `include_type_name` set to `false`. + +In case typeless templates are used with typed index creation calls or typed +templates are used with typeless index creation calls, the template will still +be applied but the index creation call decides whether there should be a type +or not. For instance in the below example, `index-1-01` will have a type in +spite of the fact that it matches a template that is typeless, and `index-2-01` +will be typeless in spite of the fact that it matches a template that defines +a type. Both `index-1-01` and `index-2-01` will inherit the `foo` field from +the template that they match. + +[source,js] +-------------------------------------------------- +PUT _template/template1 +{ + "index_patterns":[ "index-1-*" ], + "mappings": { + "properties": { + "foo": { + "type": "keyword" + } + } + } +} + +PUT _template/template2?include_type_name=true +{ + "index_patterns":[ "index-2-*" ], + "mappings": { + "type": { + "properties": { + "foo": { + "type": "keyword" + } + } + } + } +} + +PUT index-1-01?include_type_name=true +{ + "mappings": { + "type": { + "properties": { + "bar": { + "type": "long" + } + } + } + } +} + +PUT index-2-01 +{ + "mappings": { + "properties": { + "bar": { + "type": "long" + } + } + } +} +-------------------------------------------------- +// CONSOLE + +In case of implicit index creation, because of documents that get indexed in +an index that doesn't exist yet, the template is always honored. This is +usually not a problem due to the fact that typless index calls work on typed +indices. diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/20_mix_typeless_typeful.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/20_mix_typeless_typeful.yml new file mode 100644 index 0000000000000..d196d5e5dd8c2 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/20_mix_typeless_typeful.yml @@ -0,0 +1,137 @@ +--- +"Create a typeless index while there is a typed template": + + - skip: + version: " - 6.99.99" + reason: needs change to be backported to 6.7 + + - do: + indices.put_template: + include_type_name: true + name: test_template + body: + index_patterns: test-* + mappings: + my_type: + properties: + foo: + type: keyword + + - do: + indices.create: + include_type_name: false + index: test-1 + body: + mappings: + properties: + bar: + type: "long" + + - do: + indices.get_mapping: + include_type_name: true + index: test-1 + + - is_true: test-1.mappings._doc # the index creation call won + - is_false: test-1.mappings.my_type + - is_true: test-1.mappings._doc.properties.foo + - is_true: test-1.mappings._doc.properties.bar + +--- +"Create a typed index while there is a typeless template": + + - skip: + version: " - 6.99.99" + reason: needs change to be backported to 6.7 + + - do: + indices.put_template: + include_type_name: false + name: test_template + body: + index_patterns: test-* + mappings: + properties: + foo: + type: keyword + + - do: + indices.create: + include_type_name: true + index: test-1 + body: + mappings: + my_type: + properties: + bar: + type: "long" + + - do: + indices.get_mapping: + include_type_name: true + index: test-1 + + - is_true: test-1.mappings.my_type # the index creation call won + - is_false: test-1.mappings._doc + - is_true: test-1.mappings.my_type.properties.foo + - is_true: test-1.mappings.my_type.properties.bar + +--- +"Implicitly create a typed index while there is a typeless template": + + - skip: + version: " - 6.99.99" + reason: needs change to be backported to 6.7 + + - do: + indices.put_template: + include_type_name: false + name: test_template + body: + index_patterns: test-* + mappings: + properties: + foo: + type: keyword + + - do: + catch: /the final mapping would have more than 1 type/ + index: + index: test-1 + type: my_type + body: { bar: 42 } + +--- +"Implicitly create a typeless index while there is a typed template": + + - skip: + version: " - 6.99.99" + reason: needs typeless index operations to work on typed indices + + - do: + indices.put_template: + include_type_name: true + name: test_template + body: + index_patterns: test-* + mappings: + my_type: + properties: + foo: + type: keyword + + - do: + index: + index: test-1 + type: my_type + body: { bar: 42 } + + - do: + indices.get_mapping: + include_type_name: true + index: test-1 + + - is_true: test-1.mappings.my_type # the template is honored + - is_false: test-1.mappings._doc + - is_true: test-1.mappings.my_type.properties.foo + - is_true: test-1.mappings.my_type.properties.bar diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index d9120342cf4cd..838b1e2547204 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -318,6 +318,28 @@ public ClusterState execute(ClusterState currentState) throws Exception { if (mappings.containsKey(cursor.key)) { XContentHelper.mergeDefaults(mappings.get(cursor.key), MapperService.parseMapping(xContentRegistry, mappingString)); + } else if (mappings.size() == 1 && cursor.key.equals(MapperService.SINGLE_MAPPING_NAME)) { + // Typeless template with typed mapping + Map templateMapping = MapperService.parseMapping(xContentRegistry, mappingString); + assert templateMapping.size() == 1 : templateMapping; + assert cursor.key.equals(templateMapping.keySet().iterator().next()) : + cursor.key + " != " + templateMapping; + Map.Entry> mappingEntry = mappings.entrySet().iterator().next(); + templateMapping = Collections.singletonMap( + mappingEntry.getKey(), // reuse type name from the mapping + templateMapping.values().iterator().next()); // but actual mappings from the template + XContentHelper.mergeDefaults(mappingEntry.getValue(), templateMapping); + } else if (template.mappings().size() == 1 && mappings.containsKey(MapperService.SINGLE_MAPPING_NAME)) { + // Typed template with typeless mapping + Map templateMapping = MapperService.parseMapping(xContentRegistry, mappingString); + assert templateMapping.size() == 1 : templateMapping; + assert cursor.key.equals(templateMapping.keySet().iterator().next()) : + cursor.key + " != " + templateMapping; + Map mapping = mappings.get(MapperService.SINGLE_MAPPING_NAME); + templateMapping = Collections.singletonMap( + MapperService.SINGLE_MAPPING_NAME, // make template mapping typeless + templateMapping.values().iterator().next()); + XContentHelper.mergeDefaults(mapping, templateMapping); } else { mappings.put(cursor.key, MapperService.parseMapping(xContentRegistry, mappingString)); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java index 518a60ffe38f6..f2a87d09eb1f2 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java @@ -310,6 +310,32 @@ public void testWriteIndexValidationException() throws Exception { assertThat(exception.getMessage(), startsWith("alias [alias1] has more than one write index [")); } + public void testTypelessTemplateWithTypedIndexCreation() throws Exception { + addMatchingTemplate(builder -> builder.putMapping("type", "{\"type\": {}}")); + setupRequestMapping(MapperService.SINGLE_MAPPING_NAME, new CompressedXContent("{\"_doc\":{}}")); + executeTask(); + assertThat(getMappingsFromResponse(), Matchers.hasKey(MapperService.SINGLE_MAPPING_NAME)); + } + + public void testTypedTemplateWithTypelessIndexCreation() throws Exception { + addMatchingTemplate(builder -> builder.putMapping(MapperService.SINGLE_MAPPING_NAME, "{\"_doc\": {}}")); + setupRequestMapping("type", new CompressedXContent("{\"type\":{}}")); + executeTask(); + assertThat(getMappingsFromResponse(), Matchers.hasKey("type")); + } + + public void testTypedTemplate() throws Exception { + addMatchingTemplate(builder -> builder.putMapping("type", "{\"type\": {}}")); + executeTask(); + assertThat(getMappingsFromResponse(), Matchers.hasKey("type")); + } + + public void testTypelessTemplate() throws Exception { + addMatchingTemplate(builder -> builder.putMapping(MapperService.SINGLE_MAPPING_NAME, "{\"_doc\": {}}")); + executeTask(); + assertThat(getMappingsFromResponse(), Matchers.hasKey(MapperService.SINGLE_MAPPING_NAME)); + } + private IndexRoutingTable createIndexRoutingTableWithStartedShards(Index index) { final IndexRoutingTable idxRoutingTable = mock(IndexRoutingTable.class);