diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index fe6616cb4fb8e..c1605b06e9858 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -32,6 +32,7 @@ import org.elasticsearch.index.engine.EngineConfig; import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.store.FsDirectoryFactory; @@ -181,6 +182,8 @@ public final class IndexScopedSettings extends AbstractScopedSettings { IndexSettings.TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING, IndexSettings.PREFER_ILM_SETTING, DataStreamFailureStoreDefinition.FAILURE_STORE_DEFINITION_VERSION_SETTING, + IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING, + IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING, // validate that built-in similarities don't get redefined Setting.groupSetting("index.similarity.", (s) -> { diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 944d50f7ea06c..ed0fc829eb2fe 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.ingest.IngestService; import org.elasticsearch.node.Node; @@ -776,6 +777,8 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) { private volatile long mappingDepthLimit; private volatile long mappingFieldNameLengthLimit; private volatile long mappingDimensionFieldsLimit; + private volatile boolean skipIgnoredSourceWrite; + private volatile boolean skipIgnoredSourceRead; /** * The maximum number of refresh listeners allows on this shard. @@ -923,6 +926,8 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti mappingDimensionFieldsLimit = scopedSettings.get(INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING); indexRouting = IndexRouting.fromIndexMetadata(indexMetadata); es87TSDBCodecEnabled = scopedSettings.get(TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING); + skipIgnoredSourceWrite = scopedSettings.get(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING); + skipIgnoredSourceRead = scopedSettings.get(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING); scopedSettings.addSettingsUpdateConsumer( MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING, @@ -1005,6 +1010,11 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_DEPTH_LIMIT_SETTING, this::setMappingDepthLimit); scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING, this::setMappingFieldNameLengthLimit); scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING, this::setMappingDimensionFieldsLimit); + scopedSettings.addSettingsUpdateConsumer( + IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING, + this::setSkipIgnoredSourceWrite + ); + scopedSettings.addSettingsUpdateConsumer(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING, this::setSkipIgnoredSourceRead); } private void setSearchIdleAfter(TimeValue searchIdleAfter) { @@ -1581,6 +1591,22 @@ private void setMappingDimensionFieldsLimit(long value) { this.mappingDimensionFieldsLimit = value; } + public boolean getSkipIgnoredSourceWrite() { + return skipIgnoredSourceWrite; + } + + private void setSkipIgnoredSourceWrite(boolean value) { + this.skipIgnoredSourceWrite = value; + } + + public boolean getSkipIgnoredSourceRead() { + return skipIgnoredSourceRead; + } + + private void setSkipIgnoredSourceRead(boolean value) { + this.skipIgnoredSourceRead = value; + } + /** * The bounds for {@code @timestamp} on this index or * {@code null} if there are no bounds. diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index d8fa2919b795f..5f73a7e618713 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -326,7 +326,7 @@ final boolean getClonedSource() { } public final boolean canAddIgnoredField() { - return mappingLookup.isSourceSynthetic() && clonedSource == false; + return mappingLookup.isSourceSynthetic() && clonedSource == false && indexSettings().getSkipIgnoredSourceWrite() == false; } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java index f94a05b2a8658..debaa394f2384 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java @@ -9,12 +9,15 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.LeafReader; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.util.ByteUtils; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Tuple; import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; @@ -26,6 +29,8 @@ import java.util.Comparator; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; /** @@ -39,6 +44,7 @@ * if we can replace it for all use cases to avoid duplication, assuming that the storage tradeoff is favorable. */ public class IgnoredSourceFieldMapper extends MetadataFieldMapper { + private final IndexSettings indexSettings; // This factor is used to combine two offsets within the same integer: // - the offset of the end of the parent field within the field name (N / PARENT_OFFSET_IN_NAME_OFFSET) @@ -48,12 +54,32 @@ public class IgnoredSourceFieldMapper extends MetadataFieldMapper { public static final String NAME = "_ignored_source"; - public static final IgnoredSourceFieldMapper INSTANCE = new IgnoredSourceFieldMapper(); - - public static final TypeParser PARSER = new FixedTypeParser(context -> INSTANCE); + public static final TypeParser PARSER = new FixedTypeParser(context -> new IgnoredSourceFieldMapper(context.getIndexSettings())); static final NodeFeature TRACK_IGNORED_SOURCE = new NodeFeature("mapper.track_ignored_source"); + /* + Setting to disable encoding and writing values for this field. + This is needed to unblock index functionality in case there is a bug on this code path. + */ + public static final Setting SKIP_IGNORED_SOURCE_WRITE_SETTING = Setting.boolSetting( + "index.mapping.synthetic_source.skip_ignored_source_write", + false, + Setting.Property.Dynamic, + Setting.Property.IndexScope + ); + + /* + Setting to disable reading and decoding values stored in this field. + This is needed to unblock search functionality in case there is a bug on this code path. + */ + public static final Setting SKIP_IGNORED_SOURCE_READ_SETTING = Setting.boolSetting( + "index.mapping.synthetic_source.skip_ignored_source_read", + false, + Setting.Property.Dynamic, + Setting.Property.IndexScope + ); + /* * Container for the ignored field data: * - the full name @@ -107,8 +133,9 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) } } - private IgnoredSourceFieldMapper() { + private IgnoredSourceFieldMapper(IndexSettings indexSettings) { super(IgnoredValuesFieldMapperType.INSTANCE); + this.indexSettings = indexSettings; } @Override @@ -150,6 +177,18 @@ static NameValue decode(Object field) { return new NameValue(name, parentOffset, value, null); } + // In rare cases decoding values stored in this field can fail leading to entire source + // not being available. + // We would like to have an option to lose some values in synthetic source + // but have search not fail. + public static Set ensureLoaded(Set fieldsToLoadForSyntheticSource, IndexSettings indexSettings) { + if (indexSettings.getSkipIgnoredSourceRead() == false) { + fieldsToLoadForSyntheticSource.add(NAME); + } + + return fieldsToLoadForSyntheticSource; + } + public record MappedNameValue(NameValue nameValue, XContentType type, Map map) {} /** @@ -200,11 +239,49 @@ public static byte[] encodeFromMap(MappedNameValue mappedNameValue, Map> storedFieldLoaders() { + if (indexSettings.getSkipIgnoredSourceRead()) { + return Stream.empty(); + } + + // Values are handled in `SourceLoader`. + return Stream.of(Map.entry(NAME, (v) -> {})); + } + + @Override + public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf) throws IOException { + return null; + } + + @Override + public boolean hasValue() { + return false; + } + + @Override + public void write(XContentBuilder b) throws IOException { + + } + + @Override + public String fieldName() { + // Does not really matter. + return NAME; + } + + @Override + public void reset() { + + } + }; } - } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java index f9842501fc69e..d300003da6bb9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.Explicit; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; @@ -46,11 +47,18 @@ public static class Builder extends ObjectMapper.Builder { private Explicit includeInParent = Explicit.IMPLICIT_FALSE; private final IndexVersion indexCreatedVersion; private final Function bitSetProducer; + private final IndexSettings indexSettings; - public Builder(String name, IndexVersion indexCreatedVersion, Function bitSetProducer) { + public Builder( + String name, + IndexVersion indexCreatedVersion, + Function bitSetProducer, + IndexSettings indexSettings + ) { super(name, Explicit.IMPLICIT_TRUE); this.indexCreatedVersion = indexCreatedVersion; this.bitSetProducer = bitSetProducer; + this.indexSettings = indexSettings; } Builder includeInRoot(boolean includeInRoot) { @@ -111,7 +119,8 @@ public NestedObjectMapper build(MapperBuilderContext context) { parentTypeFilter, nestedTypePath, nestedTypeFilter, - bitSetProducer + bitSetProducer, + indexSettings ); } } @@ -126,7 +135,8 @@ public Mapper.Builder parse(String name, Map node, MappingParser NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder( name, parserContext.indexVersionCreated(), - parserContext::bitSetProducer + parserContext::bitSetProducer, + parserContext.getIndexSettings() ); parseNested(name, node, builder); parseObjectFields(node, parserContext, builder); @@ -193,6 +203,7 @@ public MapperBuilderContext createChildContext(String name, Dynamic dynamic) { private final Query nestedTypeFilter; // Function to create a bitset for identifying parent documents private final Function bitsetProducer; + private final IndexSettings indexSettings; NestedObjectMapper( String name, @@ -206,7 +217,8 @@ public MapperBuilderContext createChildContext(String name, Dynamic dynamic) { Query parentTypeFilter, String nestedTypePath, Query nestedTypeFilter, - Function bitsetProducer + Function bitsetProducer, + IndexSettings indexSettings ) { super(name, fullPath, enabled, Explicit.IMPLICIT_TRUE, storeArraySource, dynamic, mappers); this.parentTypeFilter = parentTypeFilter; @@ -215,6 +227,7 @@ public MapperBuilderContext createChildContext(String name, Dynamic dynamic) { this.includeInParent = includeInParent; this.includeInRoot = includeInRoot; this.bitsetProducer = bitsetProducer; + this.indexSettings = indexSettings; } public Query parentTypeFilter() { @@ -252,7 +265,7 @@ public Map getChildren() { @Override public ObjectMapper.Builder newBuilder(IndexVersion indexVersionCreated) { - NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder(leafName(), indexVersionCreated, bitsetProducer); + NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder(leafName(), indexVersionCreated, bitsetProducer, indexSettings); builder.enabled = enabled; builder.dynamic = dynamic; builder.includeInRoot = includeInRoot; @@ -274,7 +287,8 @@ NestedObjectMapper withoutMappers() { parentTypeFilter, nestedTypePath, nestedTypeFilter, - bitsetProducer + bitsetProducer, + indexSettings ); } @@ -349,7 +363,8 @@ public ObjectMapper merge(Mapper mergeWith, MapperMergeContext parentMergeContex parentTypeFilter, nestedTypePath, nestedTypeFilter, - bitsetProducer + bitsetProducer, + indexSettings ); } @@ -382,7 +397,9 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { } SourceLoader sourceLoader = new SourceLoader.Synthetic(() -> super.syntheticFieldLoader(mappers.values().stream(), true), NOOP); - var storedFieldLoader = org.elasticsearch.index.fieldvisitor.StoredFieldLoader.create(false, sourceLoader.requiredStoredFields()); + // Some synthetic source use cases require using _ignored_source field + var requiredStoredFields = IgnoredSourceFieldMapper.ensureLoaded(sourceLoader.requiredStoredFields(), indexSettings); + var storedFieldLoader = org.elasticsearch.index.fieldvisitor.StoredFieldLoader.create(false, requiredStoredFields); return new NestedSyntheticFieldLoader( storedFieldLoader, sourceLoader, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java index 319fd7fc792ef..c046067e44571 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceLoader.java @@ -117,7 +117,6 @@ public Synthetic(Supplier fieldLoaderSupplier, SourceField .storedFieldLoaders() .map(Map.Entry::getKey) .collect(Collectors.toSet()); - this.requiredStoredFields.add(IgnoredSourceFieldMapper.NAME); this.metrics = metrics; } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java index d913b86aed2d5..ede96e7230cf5 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldAliasMapperValidationTests.java @@ -184,9 +184,8 @@ private static ObjectMapper createObjectMapper(String name) { } private static NestedObjectMapper createNestedObjectMapper(String name) { - return new NestedObjectMapper.Builder(name, IndexVersion.current(), query -> { throw new UnsupportedOperationException(); }).build( - MapperBuilderContext.root(false, false) - ); + return new NestedObjectMapper.Builder(name, IndexVersion.current(), query -> { throw new UnsupportedOperationException(); }, null) + .build(MapperBuilderContext.root(false, false)); } private static MappingLookup createMappingLookup( diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java new file mode 100644 index 0000000000000..b24bfeab330b6 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java @@ -0,0 +1,143 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.index.DirectoryReader; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Map; + +public class IgnoredSourceFieldMapperConfigurationTests extends MapperServiceTestCase { + public void testDisableIgnoredSourceRead() throws IOException { + var mapperService = mapperServiceWithCustomSettings( + Map.of(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING.getKey(), true), + b -> { + b.startObject("fallback_field"); + { + b.field("type", "long").field("doc_values", "false"); + } + b.endObject(); + b.startObject("disabled_object"); + { + b.field("enabled", "false"); + b.startObject("properties"); + { + b.startObject("field").field("type", "keyword").endObject(); + } + b.endObject(); + } + b.endObject(); + } + ); + + CheckedConsumer inputDocument = b -> { + b.field("fallback_field", 111); + b.startObject("disabled_object"); + { + b.field("field", "hey"); + } + b.endObject(); + }; + + var doc = mapperService.documentMapper().parse(source(inputDocument)); + // Field was written. + assertNotNull(doc.docs().get(0).getField(IgnoredSourceFieldMapper.NAME)); + + String syntheticSource = syntheticSource(mapperService.documentMapper(), inputDocument); + // Values are not loaded. + assertEquals("{}", syntheticSource); + + mapperService.getIndexSettings() + .getScopedSettings() + .applySettings(Settings.builder().put(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING.getKey(), false).build()); + + doc = mapperService.documentMapper().parse(source(inputDocument)); + // Field was written. + assertNotNull(doc.docs().get(0).getField(IgnoredSourceFieldMapper.NAME)); + + syntheticSource = syntheticSource(mapperService.documentMapper(), inputDocument); + // Values are loaded. + assertEquals("{\"disabled_object\":{\"field\":\"hey\"},\"fallback_field\":111}", syntheticSource); + } + + public void testDisableIgnoredSourceWrite() throws IOException { + var mapperService = mapperServiceWithCustomSettings( + Map.of(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING.getKey(), true), + b -> { + b.startObject("fallback_field"); + { + b.field("type", "long").field("doc_values", "false"); + } + b.endObject(); + b.startObject("disabled_object"); + { + b.field("enabled", "false"); + b.startObject("properties"); + { + b.startObject("field").field("type", "keyword").endObject(); + } + b.endObject(); + } + b.endObject(); + } + ); + + CheckedConsumer inputDocument = b -> { + b.field("fallback_field", 111); + b.startObject("disabled_object"); + { + b.field("field", "hey"); + } + b.endObject(); + }; + + var doc = mapperService.documentMapper().parse(source(inputDocument)); + // Field is not written. + assertNull(doc.docs().get(0).getField(IgnoredSourceFieldMapper.NAME)); + + String syntheticSource = syntheticSource(mapperService.documentMapper(), inputDocument); + // Values are not loaded. + assertEquals("{}", syntheticSource); + + mapperService.getIndexSettings() + .getScopedSettings() + .applySettings(Settings.builder().put(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING.getKey(), false).build()); + + doc = mapperService.documentMapper().parse(source(inputDocument)); + // Field was written. + assertNotNull(doc.docs().get(0).getField(IgnoredSourceFieldMapper.NAME)); + + syntheticSource = syntheticSource(mapperService.documentMapper(), inputDocument); + // Values are loaded. + assertEquals("{\"disabled_object\":{\"field\":\"hey\"},\"fallback_field\":111}", syntheticSource); + } + + private MapperService mapperServiceWithCustomSettings( + Map customSettings, + CheckedConsumer mapping + ) throws IOException { + var settings = Settings.builder(); + for (var entry : customSettings.entrySet()) { + settings.put(entry.getKey(), entry.getValue()); + } + + return createMapperService(settings.build(), syntheticSourceMapping(mapping)); + } + + protected void validateRoundTripReader(String syntheticSource, DirectoryReader reader, DirectoryReader roundTripReader) + throws IOException { + // Disabling this field via index settings leads to some values not being present in source and assertReaderEquals validation to + // fail as a result. + // This is expected, these settings are introduced only as a safety net when related logic blocks ingestion or search + // and we would rather lose some part of source but unblock the workflow. + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java.new b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java.new new file mode 100644 index 0000000000000..9a771613bf3b4 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java.new @@ -0,0 +1,164 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +import org.apache.lucene.index.DirectoryReader; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Map; + +public class IgnoredSourceFieldMapperConfigurationTests extends MapperServiceTestCase { + public void testDisableIgnoredSourceRead() throws IOException { + var mapperService = mapperServiceWithCustomSettings( + Map.of(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING.getKey(), true), + b -> { + b.startObject("fallback_field"); + { + b.field("type", "long").field("doc_values", "false"); + } + b.endObject(); + b.startObject("disabled_object"); + { + b.field("enabled", "false"); + b.startObject("properties"); + { + b.startObject("field").field("type", "keyword").endObject(); + } + b.endObject(); + } + b.endObject(); + } + ); + + CheckedConsumer inputDocument = b -> { + b.field("fallback_field", 111); + b.startObject("disabled_object"); + { + b.field("field", "hey"); + } + b.endObject(); + }; + + var doc = mapperService.documentMapper().parse(source(inputDocument)); + // Field was written. + assertNotNull(doc.docs().get(0).getField(IgnoredSourceFieldMapper.NAME)); + + String syntheticSource = syntheticSource(mapperService.documentMapper(), inputDocument); + // Values are not loaded. + assertEquals("{}", syntheticSource); + + mapperService.getIndexSettings() + .getScopedSettings() + .applySettings(Settings.builder().put(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING.getKey(), false).build()); + + doc = mapperService.documentMapper().parse(source(inputDocument)); + // Field was written. + assertNotNull(doc.docs().get(0).getField(IgnoredSourceFieldMapper.NAME)); + + syntheticSource = syntheticSource(mapperService.documentMapper(), inputDocument); + // Values are loaded. + assertEquals("{\"disabled_object\":{\"field\":\"hey\"},\"fallback_field\":111}", syntheticSource); + } + + public void testDisableIgnoredSourceWrite() throws IOException { + var mapperService = mapperServiceWithCustomSettings( + Map.of(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING.getKey(), true), + b -> { + b.startObject("fallback_field"); + { + b.field("type", "long").field("doc_values", "false"); + } + b.endObject(); + b.startObject("disabled_object"); + { + b.field("enabled", "false"); + b.startObject("properties"); + { + b.startObject("field").field("type", "keyword").endObject(); + } + b.endObject(); + } + b.endObject(); + } + ); + + CheckedConsumer inputDocument = b -> { + b.field("fallback_field", 111); + b.startObject("disabled_object"); + { + b.field("field", "hey"); + } + b.endObject(); + }; + + var doc = mapperService.documentMapper().parse(source(inputDocument)); + // Field is not written. + assertNull(doc.docs().get(0).getField(IgnoredSourceFieldMapper.NAME)); + + String syntheticSource = syntheticSource(mapperService.documentMapper(), inputDocument); + // Values are not loaded. + assertEquals("{}", syntheticSource); + + mapperService.getIndexSettings() + .getScopedSettings() + .applySettings(Settings.builder().put(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING.getKey(), false).build()); + + doc = mapperService.documentMapper().parse(source(inputDocument)); + // Field was written. + assertNotNull(doc.docs().get(0).getField(IgnoredSourceFieldMapper.NAME)); + + syntheticSource = syntheticSource(mapperService.documentMapper(), inputDocument); + // Values are loaded. + assertEquals("{\"disabled_object\":{\"field\":\"hey\"},\"fallback_field\":111}", syntheticSource); + } + + private MapperService mapperServiceWithCustomSettings( + Map customSettings, + CheckedConsumer mapping + ) throws IOException { + var settings = Settings.builder(); + for (var entry : customSettings.entrySet()) { + settings.put(entry.getKey(), entry.getValue()); + } + + return createMapperService(settings.build(), syntheticSourceMapping(mapping)); + } + + protected void validateRoundTripReader(String syntheticSource, DirectoryReader reader, DirectoryReader roundTripReader) + throws IOException { + // Disabling this field via index settings leads to some values not being present in source and assertReaderEquals validation to + // fail as a result. + // This is expected, these settings are introduced only as a safety net when related logic blocks ingestion or search + // and we would rather lose some part of source but unblock the workflow. + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NestedLookupTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NestedLookupTests.java index 5c2fa6e89b0c6..ed65a6e4650bd 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NestedLookupTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NestedLookupTests.java @@ -64,9 +64,8 @@ public void testMultiLevelParents() throws IOException { } private static NestedObjectMapper buildMapper(String name) { - return new NestedObjectMapper.Builder(name, IndexVersion.current(), query -> { throw new UnsupportedOperationException(); }).build( - MapperBuilderContext.root(false, false) - ); + return new NestedObjectMapper.Builder(name, IndexVersion.current(), query -> { throw new UnsupportedOperationException(); }, null) + .build(MapperBuilderContext.root(false, false)); } public void testAllParentFilters() { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java index 13bd5955d67a5..9b5d9c4dcec5d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java @@ -1504,10 +1504,10 @@ public void testIndexTemplatesMergeIncludes() throws IOException { public void testMergeNested() { NestedObjectMapper firstMapper = new NestedObjectMapper.Builder("nested1", IndexVersion.current(), query -> { throw new UnsupportedOperationException(); - }).includeInParent(true).includeInRoot(true).build(MapperBuilderContext.root(false, false)); + }, null).includeInParent(true).includeInRoot(true).build(MapperBuilderContext.root(false, false)); NestedObjectMapper secondMapper = new NestedObjectMapper.Builder("nested1", IndexVersion.current(), query -> { throw new UnsupportedOperationException(); - }).includeInParent(false).includeInRoot(true).build(MapperBuilderContext.root(false, false)); + }, null).includeInParent(false).includeInRoot(true).build(MapperBuilderContext.root(false, false)); MapperException e = expectThrows( MapperException.class, @@ -1854,7 +1854,7 @@ public void testNestedMapperBuilderContextConstructor() { MergeReason mergeReason = randomFrom(MergeReason.values()); MapperBuilderContext mapperBuilderContext = MapperBuilderContext.root(isSourceSynthetic, isDataStream, mergeReason); mapperBuilderContext = mapperBuilderContext.createChildContext("name", parentContainsDimensions, randomFrom(Dynamic.values())); - NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder("name", IndexVersion.current(), query -> null); + NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder("name", IndexVersion.current(), query -> null, null); builder.add(new Mapper.Builder("name") { @Override public Mapper build(MapperBuilderContext context) { @@ -1875,7 +1875,7 @@ public void testNestedMapperMergeContextRootConstructor() { MergeReason mergeReason = randomFrom(MergeReason.values()); { MapperBuilderContext mapperBuilderContext = MapperBuilderContext.root(false, false, mergeReason); - NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder("name", IndexVersion.current(), query -> null); + NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder("name", IndexVersion.current(), query -> null, null); NestedObjectMapper nestedObjectMapper = builder.build(mapperBuilderContext); MapperMergeContext mapperMergeContext = MapperMergeContext.root(isSourceSynthetic, isDataStream, mergeReason, randomLong()); MapperMergeContext childMergeContext = nestedObjectMapper.createChildContext(mapperMergeContext, "name"); @@ -1906,7 +1906,7 @@ public void testNestedMapperMergeContextFromConstructor() { MergeReason mergeReason = randomFrom(MergeReason.values()); MapperBuilderContext mapperBuilderContext = MapperBuilderContext.root(isSourceSynthetic, isDataStream, mergeReason); mapperBuilderContext = mapperBuilderContext.createChildContext("name", parentContainsDimensions, randomFrom(Dynamic.values())); - NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder("name", IndexVersion.current(), query -> null); + NestedObjectMapper.Builder builder = new NestedObjectMapper.Builder("name", IndexVersion.current(), query -> null, null); NestedObjectMapper nestedObjectMapper = builder.build(mapperBuilderContext); MapperMergeContext mapperMergeContext = MapperMergeContext.from(mapperBuilderContext, randomLong()); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java index 91078c9babe3d..c807792c994a8 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java @@ -912,8 +912,7 @@ protected List objectMappers() { ); public static NestedObjectMapper nestedObject(String path) { - return new NestedObjectMapper.Builder(path, IndexVersion.current(), query -> { throw new UnsupportedOperationException(); }).build( - MapperBuilderContext.root(false, false) - ); + return new NestedObjectMapper.Builder(path, IndexVersion.current(), query -> { throw new UnsupportedOperationException(); }, null) + .build(MapperBuilderContext.root(false, false)); } } diff --git a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index f5fbca13db1db..411fc6a341d3d 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -196,7 +196,7 @@ protected final SearchExecutionContext createMockSearchExecutionContext(IndexSea }; NestedLookup nestedLookup = NestedLookup.build(List.of(new NestedObjectMapper.Builder("path", IndexVersion.current(), query -> { throw new UnsupportedOperationException(); - }).build(MapperBuilderContext.root(false, false)))); + }, null).build(MapperBuilderContext.root(false, false)))); return new SearchExecutionContext( 0, 0, diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java index 7de0d775ba150..b4be0b33a464e 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.MapperTestUtils; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.ccr.Ccr; @@ -331,6 +332,8 @@ public void testDynamicIndexSettingsAreClassified() { replicatedSettings.add(IndexSettings.MAX_SHINGLE_DIFF_SETTING); replicatedSettings.add(IndexSettings.TIME_SERIES_END_TIME); replicatedSettings.add(IndexSettings.PREFER_ILM_SETTING); + replicatedSettings.add(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING); + replicatedSettings.add(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING); for (Setting setting : IndexScopedSettings.BUILT_IN_INDEX_SETTINGS) { // removed settings have no effect, they are only there for BWC diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java index b9b95afbf6dc6..e224f38621f56 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Tuple; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; @@ -79,13 +80,16 @@ public class SemanticTextFieldMapper extends FieldMapper implements InferenceFieldMapper { public static final String CONTENT_TYPE = "semantic_text"; + private final IndexSettings indexSettings; + public static final TypeParser PARSER = new TypeParser( - (n, c) -> new Builder(n, c.indexVersionCreated(), c::bitSetProducer), + (n, c) -> new Builder(n, c.indexVersionCreated(), c::bitSetProducer, c.getIndexSettings()), List.of(notInMultiFields(CONTENT_TYPE), notFromDynamicTemplates(CONTENT_TYPE)) ); public static class Builder extends FieldMapper.Builder { private final IndexVersion indexVersionCreated; + private final IndexSettings indexSettings; private final Parameter inferenceId = Parameter.stringParam( "inference_id", @@ -112,10 +116,22 @@ public static class Builder extends FieldMapper.Builder { private Function inferenceFieldBuilder; - public Builder(String name, IndexVersion indexVersionCreated, Function bitSetProducer) { + public Builder( + String name, + IndexVersion indexVersionCreated, + Function bitSetProducer, + IndexSettings indexSettings + ) { super(name); this.indexVersionCreated = indexVersionCreated; - this.inferenceFieldBuilder = c -> createInferenceField(c, indexVersionCreated, modelSettings.get(), bitSetProducer); + this.indexSettings = indexSettings; + this.inferenceFieldBuilder = c -> createInferenceField( + c, + indexVersionCreated, + modelSettings.get(), + bitSetProducer, + indexSettings + ); } public Builder setInferenceId(String id) { @@ -169,13 +185,15 @@ public SemanticTextFieldMapper build(MapperBuilderContext context) { indexVersionCreated, meta.getValue() ), - copyTo + copyTo, + indexSettings ); } } - private SemanticTextFieldMapper(String simpleName, MappedFieldType mappedFieldType, CopyTo copyTo) { + private SemanticTextFieldMapper(String simpleName, MappedFieldType mappedFieldType, CopyTo copyTo, IndexSettings indexSettings) { super(simpleName, mappedFieldType, MultiFields.empty(), copyTo); + this.indexSettings = indexSettings; } @Override @@ -187,7 +205,9 @@ public Iterator iterator() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), fieldType().indexVersionCreated, fieldType().getChunksField().bitsetProducer()).init(this); + return new Builder(leafName(), fieldType().indexVersionCreated, fieldType().getChunksField().bitsetProducer(), indexSettings).init( + this + ); } @Override @@ -228,7 +248,8 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio Builder builder = (Builder) new Builder( leafName(), fieldType().indexVersionCreated, - fieldType().getChunksField().bitsetProducer() + fieldType().getChunksField().bitsetProducer(), + indexSettings ).init(this); try { mapper = builder.setModelSettings(field.inference().modelSettings()) @@ -467,19 +488,26 @@ private static ObjectMapper createInferenceField( MapperBuilderContext context, IndexVersion indexVersionCreated, @Nullable SemanticTextField.ModelSettings modelSettings, - Function bitSetProducer + Function bitSetProducer, + IndexSettings indexSettings ) { return new ObjectMapper.Builder(INFERENCE_FIELD, Explicit.EXPLICIT_TRUE).dynamic(ObjectMapper.Dynamic.FALSE) - .add(createChunksField(indexVersionCreated, modelSettings, bitSetProducer)) + .add(createChunksField(indexVersionCreated, modelSettings, bitSetProducer, indexSettings)) .build(context); } private static NestedObjectMapper.Builder createChunksField( IndexVersion indexVersionCreated, @Nullable SemanticTextField.ModelSettings modelSettings, - Function bitSetProducer + Function bitSetProducer, + IndexSettings indexSettings ) { - NestedObjectMapper.Builder chunksField = new NestedObjectMapper.Builder(CHUNKS_FIELD, indexVersionCreated, bitSetProducer); + NestedObjectMapper.Builder chunksField = new NestedObjectMapper.Builder( + CHUNKS_FIELD, + indexVersionCreated, + bitSetProducer, + indexSettings + ); chunksField.dynamic(ObjectMapper.Dynamic.FALSE); KeywordFieldMapper.Builder chunkTextField = new KeywordFieldMapper.Builder(CHUNKED_TEXT_FIELD, indexVersionCreated).indexed(false) .docValues(false);