Handle legacy mappings with placeholder fields (#85059)

As part of #81210 we would like to add support for handling legacy (Elasticsearch 5 and 6) mappings in newer
Elasticsearch versions. The idea is to import old mappings "as-is" into Elasticsearch 8, and adapt the mapper parsers
so that they can handle those old mappings. Only a select subset of the legacy mapping will actually be parsed, and
fields that are neither known to newer ES version nor supported for search will be mapped as "placeholder fields", i.e.,
they are still represented as fields in the system so that they can give proper error messages when queried by a user.

Fields that are supported: 

- field data types that support doc values only fields
  - normalizer on keyword fields and date formats on date fields are on supported in so far as they behave similarly
     across versions. In case they are not, these fields are now updateable on legacy indices so that they can be "fixed"
     by user.
- object fields
- nested fields in limited form (not supporting nested queries)
  - add tests / checks in follow-up PR
- multi fields
- field aliases
- metadata fields
- runtime fields (auto-import to be added for future versions)

5.x indices with mappings that have multiple mapping types are collapsed together on a best-effort basis before they
are imported.

Relates #81210

server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java

@@ -18,12 +18,15 @@
 import org.elasticsearch.common.settings.IndexScopedSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.analysis.AnalyzerScope;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
+import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.MapperRegistry;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.Mapping;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.script.ScriptCompiler;
 import org.elasticsearch.script.ScriptService;
@@ -89,7 +92,7 @@ public IndexMetadata verifyIndexMetadata(IndexMetadata indexMetadata, Version mi
         // Next we have to run this otherwise if we try to create IndexSettings
         // with broken settings it would fail in checkMappingsCompatibility
         newMetadata = archiveBrokenIndexSettings(newMetadata);
-        checkMappingsCompatibility(newMetadata);
+        createAndValidateMapping(newMetadata);
         return newMetadata;
     }
 
@@ -126,8 +129,10 @@ private static void checkSupportedVersion(IndexMetadata indexMetadata, Version m
      * Note that we don't expect users to encounter mapping incompatibilities, since our index compatibility
      * policy guarantees we can read mappings from previous compatible index versions. A failure here would
      * indicate a compatibility bug (which are unfortunately not that uncommon).
+     * @return the mapping
      */
-    private void checkMappingsCompatibility(IndexMetadata indexMetadata) {
+    @Nullable
+    public Mapping createAndValidateMapping(IndexMetadata indexMetadata) {
         try {
 
             // We cannot instantiate real analysis server or similarity service at this point because the node
@@ -194,6 +199,8 @@ public Set<Entry<String, NamedAnalyzer>> entrySet() {
                     scriptService
                 );
                 mapperService.merge(indexMetadata, MapperService.MergeReason.MAPPING_RECOVERY);
+                DocumentMapper documentMapper = mapperService.documentMapper();
+                return documentMapper == null ? null : documentMapper.mapping();
             }
         } catch (Exception ex) {
             // Wrap the inner exception so we have the index name in the exception message

server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java

@@ -142,7 +142,12 @@ private FieldValues<Boolean> scriptValues() {
         }
     }
 
-    public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.scriptCompiler(), c.indexVersionCreated()));
+    private static final Version MINIMUM_COMPATIBILITY_VERSION = Version.fromString("5.0.0");
+
+    public static final TypeParser PARSER = new TypeParser(
+        (n, c) -> new Builder(n, c.scriptCompiler(), c.indexVersionCreated()),
+        MINIMUM_COMPATIBILITY_VERSION
+    );
 
     public static final class BooleanFieldType extends TermBasedFieldType {
 

server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java

@@ -8,6 +8,9 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.document.StoredField;
@@ -72,6 +75,7 @@
 public final class DateFieldMapper extends FieldMapper {
 
     private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(DateFieldMapper.class);
+    private static final Logger logger = LogManager.getLogger(DateFieldMapper.class);
 
     public static final String CONTENT_TYPE = "date";
     public static final String DATE_NANOS_CONTENT_TYPE = "date_nanos";
@@ -266,7 +270,12 @@ public Builder(
             DateFormatter defaultFormat = resolution == Resolution.MILLISECONDS
                 ? DEFAULT_DATE_TIME_FORMATTER
                 : DEFAULT_DATE_TIME_NANOS_FORMATTER;
-            this.format = Parameter.stringParam("format", false, m -> toType(m).format, defaultFormat.pattern());
+            this.format = Parameter.stringParam(
+                "format",
+                indexCreatedVersion.isLegacyIndexVersion(),
+                m -> toType(m).format,
+                defaultFormat.pattern()
+            );
             if (dateFormatter != null) {
                 this.format.setValue(dateFormatter.pattern());
                 this.locale.setValue(dateFormatter.locale());
@@ -277,7 +286,15 @@ private DateFormatter buildFormatter() {
             try {
                 return DateFormatter.forPattern(format.getValue()).withLocale(locale.getValue());
             } catch (IllegalArgumentException e) {
-                throw new IllegalArgumentException("Error parsing [format] on field [" + name() + "]: " + e.getMessage(), e);
+                if (indexCreatedVersion.isLegacyIndexVersion()) {
+                    logger.warn(
+                        new ParameterizedMessage("Error parsing format [{}] of legacy index, falling back to default", format.getValue()),
+                        e
+                    );
+                    return DateFormatter.forPattern(format.getDefaultValue()).withLocale(locale.getValue());
+                } else {
+                    throw new IllegalArgumentException("Error parsing [format] on field [" + name() + "]: " + e.getMessage(), e);
+                }
             }
         }
 
@@ -341,6 +358,8 @@ public DateFieldMapper build(MapperBuilderContext context) {
         }
     }
 
+    private static final Version MINIMUM_COMPATIBILITY_VERSION = Version.fromString("5.0.0");
+
     public static final TypeParser MILLIS_PARSER = new TypeParser((n, c) -> {
         boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(c.getSettings());
         return new Builder(
@@ -351,7 +370,7 @@ public DateFieldMapper build(MapperBuilderContext context) {
             ignoreMalformedByDefault,
             c.indexVersionCreated()
         );
-    });
+    }, MINIMUM_COMPATIBILITY_VERSION);
 
     public static final TypeParser NANOS_PARSER = new TypeParser((n, c) -> {
         boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(c.getSettings());
@@ -363,7 +382,7 @@ public DateFieldMapper build(MapperBuilderContext context) {
             ignoreMalformedByDefault,
             c.indexVersionCreated()
         );
-    });
+    }, MINIMUM_COMPATIBILITY_VERSION);
 
     public static final class DateFieldType extends MappedFieldType {
         protected final DateFormatter dateTimeFormatter;

server/src/main/java/org/elasticsearch/index/mapper/FieldAliasMapper.java

@@ -8,6 +8,7 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.xcontent.XContentBuilder;
 
@@ -124,6 +125,11 @@ public Mapper.Builder parse(String name, Map<String, Object> node, MappingParser
             }
             return builder.path(path);
         }
+
+        @Override
+        public boolean supportsVersion(Version indexCreatedVersion) {
+            return true;
+        }
     }
 
     public static class Builder extends Mapper.Builder {

server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java

@@ -1191,7 +1191,7 @@ public Builder init(FieldMapper initializer) {
             return this;
         }
 
-        private void merge(FieldMapper in, Conflicts conflicts) {
+        protected void merge(FieldMapper in, Conflicts conflicts) {
             for (Parameter<?> param : getParameters()) {
                 param.merge(in, conflicts);
             }
@@ -1238,7 +1238,7 @@ protected void addScriptValidation(
          * Writes the current builder parameter values as XContent
          */
         @Override
-        public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             boolean includeDefaults = params.paramAsBoolean("include_defaults", false);
             for (Parameter<?> parameter : getParameters()) {
                 parameter.toXContent(builder, includeDefaults);
@@ -1304,6 +1304,12 @@ public final void parse(String name, MappingParserContext parserContext, Map<Str
                     parameter = paramsMap.get(propName);
                 }
                 if (parameter == null) {
+                    if (parserContext.indexVersionCreated().isLegacyIndexVersion()) {
+                        // ignore unknown parameters on legacy indices
+                        handleUnknownParamOnLegacyIndex(propName, propNode);
+                        iterator.remove();
+                        continue;
+                    }
                     if (isDeprecatedParameter(propName, parserContext.indexVersionCreated())) {
                         deprecationLogger.warn(
                             DeprecationCategory.API,
@@ -1352,6 +1358,10 @@ public final void parse(String name, MappingParserContext parserContext, Map<Str
             validate();
         }
 
+        protected void handleUnknownParamOnLegacyIndex(String propName, Object propNode) {
+            // ignore
+        }
+
         protected static ContentPath parentPath(String name) {
             int endPos = name.lastIndexOf(".");
             if (endPos == -1) {
@@ -1388,21 +1398,39 @@ public static final class TypeParser implements Mapper.TypeParser {
 
         private final BiFunction<String, MappingParserContext, Builder> builderFunction;
         private final BiConsumer<String, MappingParserContext> contextValidator;
+        private final Version minimumCompatibilityVersion; // see Mapper.TypeParser#supportsVersion()
 
         /**
          * Creates a new TypeParser
          * @param builderFunction a function that produces a Builder from a name and parsercontext
          */
         public TypeParser(BiFunction<String, MappingParserContext, Builder> builderFunction) {
-            this(builderFunction, (n, c) -> {});
+            this(builderFunction, (n, c) -> {}, Version.CURRENT.minimumIndexCompatibilityVersion());
+        }
+
+        /**
+         * Variant of {@link #TypeParser(BiFunction)} that allows to defining a minimumCompatibilityVersion to
+         * allow parsing mapping definitions of legacy indices (see {@link Mapper.TypeParser#supportsVersion(Version)}).
+         */
+        public TypeParser(BiFunction<String, MappingParserContext, Builder> builderFunction, Version minimumCompatibilityVersion) {
+            this(builderFunction, (n, c) -> {}, minimumCompatibilityVersion);
         }
 
         public TypeParser(
             BiFunction<String, MappingParserContext, Builder> builderFunction,
             BiConsumer<String, MappingParserContext> contextValidator
+        ) {
+            this(builderFunction, contextValidator, Version.CURRENT.minimumIndexCompatibilityVersion());
+        }
+
+        private TypeParser(
+            BiFunction<String, MappingParserContext, Builder> builderFunction,
+            BiConsumer<String, MappingParserContext> contextValidator,
+            Version minimumCompatibilityVersion
         ) {
             this.builderFunction = builderFunction;
             this.contextValidator = contextValidator;
+            this.minimumCompatibilityVersion = minimumCompatibilityVersion;
         }
 
         @Override
@@ -1412,6 +1440,11 @@ public Builder parse(String name, Map<String, Object> node, MappingParserContext
             builder.parse(name, parserContext, node);
             return builder;
         }
+
+        @Override
+        public boolean supportsVersion(Version indexCreatedVersion) {
+            return indexCreatedVersion.onOrAfter(minimumCompatibilityVersion);
+        }
     }
 
 }

server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java

@@ -162,8 +162,11 @@ public FieldMapper build(MapperBuilderContext context) {
 
     }
 
+    private static final Version MINIMUM_COMPATIBILITY_VERSION = Version.fromString("5.0.0");
+
     public static TypeParser PARSER = new TypeParser(
-        (n, c) -> new Builder(n, c.scriptCompiler(), IGNORE_MALFORMED_SETTING.get(c.getSettings()), c.indexVersionCreated())
+        (n, c) -> new Builder(n, c.scriptCompiler(), IGNORE_MALFORMED_SETTING.get(c.getSettings()), c.indexVersionCreated()),
+        MINIMUM_COMPATIBILITY_VERSION
     );
 
     private final Builder builder;

server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java

@@ -175,10 +175,12 @@ public IpFieldMapper build(MapperBuilderContext context) {
 
     }
 
+    private static final Version MINIMUM_COMPATIBILITY_VERSION = Version.fromString("5.0.0");
+
     public static final TypeParser PARSER = new TypeParser((n, c) -> {
         boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(c.getSettings());
         return new Builder(n, c.scriptCompiler(), ignoreMalformedByDefault, c.indexVersionCreated());
-    });
+    }, MINIMUM_COMPATIBILITY_VERSION);
 
     public static final class IpFieldType extends SimpleMappedFieldType {
 

server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java

@@ -8,6 +8,9 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.document.Field;
@@ -78,6 +81,8 @@
  */
 public final class KeywordFieldMapper extends FieldMapper {
 
+    private static final Logger logger = LogManager.getLogger(KeywordFieldMapper.class);
+
     public static final String CONTENT_TYPE = "keyword";
 
     public static class Defaults {
@@ -131,8 +136,7 @@ public static class Builder extends FieldMapper.Builder {
         private final Parameter<Boolean> hasNorms = TextParams.norms(false, m -> toType(m).fieldType.omitNorms() == false);
         private final Parameter<SimilarityProvider> similarity = TextParams.similarity(m -> toType(m).similarity);
 
-        private final Parameter<String> normalizer = Parameter.stringParam("normalizer", false, m -> toType(m).normalizerName, null)
-            .acceptsNull();
+        private final Parameter<String> normalizer;
 
         private final Parameter<Boolean> splitQueriesOnWhitespace = Parameter.boolParam(
             "split_queries_on_whitespace",
@@ -156,6 +160,12 @@ public Builder(String name, IndexAnalyzers indexAnalyzers, ScriptCompiler script
             this.indexAnalyzers = indexAnalyzers;
             this.scriptCompiler = Objects.requireNonNull(scriptCompiler);
             this.indexCreatedVersion = Objects.requireNonNull(indexCreatedVersion);
+            this.normalizer = Parameter.stringParam(
+                "normalizer",
+                indexCreatedVersion.isLegacyIndexVersion(),
+                m -> toType(m).normalizerName,
+                null
+            ).acceptsNull();
             this.script.precludesParameters(nullValue);
             addScriptValidation(script, indexed, hasDocValues);
 
@@ -245,7 +255,17 @@ private KeywordFieldType buildFieldType(MapperBuilderContext context, FieldType
                 assert indexAnalyzers != null;
                 normalizer = indexAnalyzers.getNormalizer(normalizerName);
                 if (normalizer == null) {
-                    throw new MapperParsingException("normalizer [" + normalizerName + "] not found for field [" + name + "]");
+                    if (indexCreatedVersion.isLegacyIndexVersion()) {
+                        logger.warn(
+                            new ParameterizedMessage(
+                                "Could not find normalizer [{}] of legacy index, falling back to default",
+                                normalizerName
+                            )
+                        );
+                        normalizer = Lucene.KEYWORD_ANALYZER;
+                    } else {
+                        throw new MapperParsingException("normalizer [" + normalizerName + "] not found for field [" + name + "]");
+                    }
                 }
                 searchAnalyzer = quoteAnalyzer = normalizer;
                 if (splitQueriesOnWhitespace.getValue()) {
@@ -274,8 +294,11 @@ public KeywordFieldMapper build(MapperBuilderContext context) {
         }
     }
 
+    private static final Version MINIMUM_COMPATIBILITY_VERSION = Version.fromString("5.0.0");
+
     public static final TypeParser PARSER = new TypeParser(
-        (n, c) -> new Builder(n, c.getIndexAnalyzers(), c.scriptCompiler(), c.indexVersionCreated())
+        (n, c) -> new Builder(n, c.getIndexAnalyzers(), c.scriptCompiler(), c.indexVersionCreated()),
+        MINIMUM_COMPATIBILITY_VERSION
     );
 
     public static final class KeywordFieldType extends StringFieldType {

server/src/main/java/org/elasticsearch/index/mapper/Mapper.java

@@ -8,6 +8,7 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.xcontent.ToXContentFragment;
 
@@ -34,6 +35,13 @@ public String name() {
 
     public interface TypeParser {
         Mapper.Builder parse(String name, Map<String, Object> node, MappingParserContext parserContext) throws MapperParsingException;
+
+        /**
+         * Whether we can parse this type on indices with the given index created version.
+         */
+        default boolean supportsVersion(Version indexCreatedVersion) {
+            return indexCreatedVersion.onOrAfter(Version.CURRENT.minimumIndexCompatibilityVersion());
+        }
     }
 
     private final String simpleName;