Adding support for simulate ingest mapping adddition for indices with…

… mappings that do not come from templates (#115359) (#115369)
elastic · Oct 22, 2024 · 9cf174f · 9cf174f
1 parent 8f568b5
commit 9cf174f
Show file tree

Hide file tree

Showing 4 changed files with 186 additions and 10 deletions.
diff --git a/docs/changelog/115359.yaml b/docs/changelog/115359.yaml
@@ -0,0 +1,6 @@
+pr: 115359
+summary: Adding support for simulate ingest mapping adddition for indices with mappings
+  that do not come from templates
+area: Ingest Node
+type: enhancement
+issues: []
diff --git a/...-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml b/...-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml
@@ -1229,7 +1229,7 @@ setup:
 
   - requires:
       cluster_features: ["simulate.mapping.addition"]
-      reason: "ingest simulate mapping addition added in 8.16"
+      reason: "ingest simulate mapping addition added in 8.17"
 
   - do:
       headers:
@@ -1465,7 +1465,7 @@ setup:
 
   - requires:
       cluster_features: ["simulate.mapping.addition"]
-      reason: "ingest simulate mapping addition added in 8.16"
+      reason: "ingest simulate mapping addition added in 8.17"
 
   - do:
       indices.put_template:
@@ -1571,3 +1571,143 @@ setup:
   - match: { docs.0.doc._source.foo: 3 }
   - match: { docs.0.doc._source.bar: "not a boolean" }
   - not_exists: docs.0.doc.error
+
+---
+"Test mapping addition works with indices without templates":
+  # In this test, we make sure that when we have an index that has mapping but was not built with a template, that the additional_mapping
+  # is merged in with that template.
+
+  - skip:
+      features:
+        - headers
+        - allowed_warnings
+
+  - requires:
+      cluster_features: ["simulate.support.non.template.mapping"]
+      reason: "ingest simulate support for indices with mappings that didn't come from templates added in 8.17"
+
+  # First, make sure that validation fails before we create the index (since we are only defining to bar field but trying to index a value
+  # for foo.
+  - do:
+      headers:
+        Content-Type: application/json
+      simulate.ingest:
+        index: foo-1
+        body: >
+          {
+            "docs": [
+              {
+                "_id": "asdf",
+                "_source": {
+                  "foo": 3,
+                  "bar": "some text value"
+                }
+              }
+            ],
+            "mapping_addition": {
+              "dynamic": "strict",
+              "properties": {
+                "bar": {
+                  "type": "keyword"
+                }
+              }
+            }
+          }
+  - length: { docs: 1 }
+  - match: { docs.0.doc._index: "foo-1" }
+  - match: { docs.0.doc._source.foo: 3 }
+  - match: { docs.0.doc._source.bar: "some text value" }
+  - match: { docs.0.doc.error.type: "strict_dynamic_mapping_exception" }
+
+  - do:
+      indices.create:
+        index: foo-1
+        body:
+          mappings:
+            dynamic: strict
+            properties:
+              foo:
+                type: integer
+  - match: { acknowledged: true }
+
+  # Now make sure that the mapping for the newly-created index is getting picked up. Validation fails because it only defined a mapping
+  # for foo, not for bar.
+  - do:
+      headers:
+        Content-Type: application/json
+      simulate.ingest:
+        index: foo-1
+        body: >
+          {
+            "docs": [
+              {
+                "_id": "asdf",
+                "_source": {
+                  "foo": 3,
+                  "bar": "some text value"
+                }
+              }
+            ]
+          }
+  - length: { docs: 1 }
+  - match: { docs.0.doc._index: "foo-1" }
+  - match: { docs.0.doc._source.foo: 3 }
+  - match: { docs.0.doc._source.bar: "some text value" }
+  - match: { docs.0.doc.error.type: "strict_dynamic_mapping_exception" }
+
+  # Now we make sure that the index's mapping gets merged with the mapping_addition:
+  - do:
+      headers:
+        Content-Type: application/json
+      simulate.ingest:
+        index: foo-1
+        body: >
+          {
+            "docs": [
+              {
+                "_id": "asdf",
+                "_source": {
+                  "foo": 3,
+                  "bar": "some text value"
+                }
+              }
+            ],
+            "mapping_addition": {
+              "dynamic": "strict",
+              "properties": {
+                "bar": {
+                  "type": "keyword"
+                }
+              }
+            }
+          }
+  - length: { docs: 1 }
+  - match: { docs.0.doc._index: "foo-1" }
+  - match: { docs.0.doc._source.foo: 3 }
+  - match: { docs.0.doc._source.bar: "some text value" }
+  - not_exists: docs.0.doc.error
+
+  # This last call to simulate is just making sure that if there are no templates, no index mappings, no substitutions, and no mapping
+  # addition, then validation does not fail
+  - do:
+      headers:
+        Content-Type: application/json
+      simulate.ingest:
+        index: nonexistent
+        body: >
+          {
+            "docs": [
+              {
+                "_id": "asdf",
+                "_source": {
+                  "foo": 3,
+                  "bar": "some text value"
+                }
+              }
+            ]
+          }
+  - length: { docs: 1 }
+  - match: { docs.0.doc._index: "nonexistent" }
+  - match: { docs.0.doc._source.foo: 3 }
+  - match: { docs.0.doc._source.bar: "some text value" }
+  - not_exists: docs.0.doc.error
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java
@@ -19,6 +19,7 @@
 import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_MAPPING_ADDITION;
 import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_MAPPING_VALIDATION;
 import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_MAPPING_VALIDATION_TEMPLATES;
+import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_SUPPORT_NON_TEMPLATE_MAPPING;
 
 public class BulkFeatures implements FeatureSpecification {
     public Set<NodeFeature> getFeatures() {
@@ -27,7 +28,8 @@ public Set<NodeFeature> getFeatures() {
             SIMULATE_MAPPING_VALIDATION_TEMPLATES,
             SIMULATE_COMPONENT_TEMPLATE_SUBSTITUTIONS,
             SIMULATE_INDEX_TEMPLATE_SUBSTITUTIONS,
-            SIMULATE_MAPPING_ADDITION
+            SIMULATE_MAPPING_ADDITION,
+            SIMULATE_SUPPORT_NON_TEMPLATE_MAPPING
         );
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java
@@ -84,6 +84,7 @@ public class TransportSimulateBulkAction extends TransportAbstractBulkAction {
     );
     public static final NodeFeature SIMULATE_INDEX_TEMPLATE_SUBSTITUTIONS = new NodeFeature("simulate.index.template.substitutions");
     public static final NodeFeature SIMULATE_MAPPING_ADDITION = new NodeFeature("simulate.mapping.addition");
+    public static final NodeFeature SIMULATE_SUPPORT_NON_TEMPLATE_MAPPING = new NodeFeature("simulate.support.non.template.mapping");
     private final IndicesService indicesService;
     private final NamedXContentRegistry xContentRegistry;
     private final Set<IndexSettingProvider> indexSettingProviders;
@@ -258,6 +259,10 @@ private Exception validateMappings(
 
                 String matchingTemplate = findV2Template(simulatedState.metadata(), request.index(), false);
                 if (matchingTemplate != null) {
+                    /*
+                     * The index matches a v2 template (including possibly one or more of the substitutions passed in). So we use this
+                     * template, and then possibly apply the mapping addition if it is not null, and validate.
+                     */
                     final Template template = TransportSimulateIndexTemplateAction.resolveTemplate(
                         matchingTemplate,
                         request.index(),
@@ -273,13 +278,36 @@ private Exception validateMappings(
                     validateUpdatedMappings(mappings, mergedMappings, request, sourceToParse);
                 } else {
                     List<IndexTemplateMetadata> matchingTemplates = findV1Templates(simulatedState.metadata(), request.index(), false);
-                    final Map<String, Object> mappingsMap = MetadataCreateIndexService.parseV1Mappings(
-                        "{}",
-                        matchingTemplates.stream().map(IndexTemplateMetadata::getMappings).collect(toList()),
-                        xContentRegistry
-                    );
-                    final CompressedXContent combinedMappings = mergeMappings(new CompressedXContent(mappingsMap), mappingAddition);
-                    validateUpdatedMappings(null, combinedMappings, request, sourceToParse);
+                    if (matchingTemplates.isEmpty() == false) {
+                        /*
+                         * The index matches v1 mappings. These are not compatible with component_template_substitutions or
+                         * index_template_substitutions, but we can apply a mapping_addition.
+                         */
+                        final Map<String, Object> mappingsMap = MetadataCreateIndexService.parseV1Mappings(
+                            "{}",
+                            matchingTemplates.stream().map(IndexTemplateMetadata::getMappings).collect(toList()),
+                            xContentRegistry
+                        );
+                        final CompressedXContent combinedMappings = mergeMappings(new CompressedXContent(mappingsMap), mappingAddition);
+                        validateUpdatedMappings(null, combinedMappings, request, sourceToParse);
+                    } else if (indexAbstraction != null && mappingAddition.isEmpty() == false) {
+                        /*
+                         * The index matched no templates of any kind, including the substitutions. But it might have a mapping. So we
+                         * merge in the mapping addition if it exists, and validate.
+                         */
+                        MappingMetadata mappingFromIndex = clusterService.state().metadata().index(indexAbstraction.getName()).mapping();
+                        CompressedXContent currentIndexCompressedXContent = mappingFromIndex == null ? null : mappingFromIndex.source();
+                        CompressedXContent combinedMappings = mergeMappings(currentIndexCompressedXContent, mappingAddition);
+                        validateUpdatedMappings(null, combinedMappings, request, sourceToParse);
+                    } else {
+                        /*
+                         * The index matched no templates and had no mapping of its own. If there were component template substitutions
+                         * or index template substitutions, they didn't match anything. So just apply the mapping addition if it exists,
+                         * and validate.
+                         */
+                        final CompressedXContent combinedMappings = mergeMappings(null, mappingAddition);
+                        validateUpdatedMappings(null, combinedMappings, request, sourceToParse);
+                    }
                 }
             }
         } catch (Exception e) {