From be3357310a4931e189444e2159d9af773b850a6e Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Tue, 29 Sep 2020 18:14:28 +0100 Subject: [PATCH] Convert all FieldMappers in mapper-extras to parametrized form (#62938) This converts RankFeatureFieldMapper, RankFeaturesFieldMapper, SearchAsYouTypeFieldMapper and TokenCountFieldMapper to parametrized forms. It also adds a TextParams utility class to core containing functions that help declare text parameters - mainly shared between SearchAsYouTypeFieldMapper and KeywordFieldMapper at the moment, but it will come in handy when we convert TextFieldMapper and friends. Relates to #62988 --- .../index/mapper/MapperExtrasPlugin.java | 8 +- .../index/mapper/RankFeatureFieldMapper.java | 72 ++----- .../index/mapper/RankFeaturesFieldMapper.java | 59 ++--- .../mapper/SearchAsYouTypeFieldMapper.java | 201 ++++++++---------- .../index/mapper/TokenCountFieldMapper.java | 148 +++++-------- .../mapper/RankFeatureFieldMapperTests.java | 23 +- .../mapper/RankFeaturesFieldMapperTests.java | 13 +- .../SearchAsYouTypeFieldMapperTests.java | 38 ++-- .../mapper/SearchAsYouTypeFieldTypeTests.java | 8 +- .../mapper/TokenCountFieldMapperTests.java | 156 ++++++-------- .../index/mapper/CompletionFieldMapper.java | 34 +-- .../index/mapper/KeywordFieldMapper.java | 24 +-- .../index/mapper/ParametrizedFieldMapper.java | 13 +- .../index/mapper/TextParams.java | 145 +++++++++++++ 14 files changed, 450 insertions(+), 492 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/TextParams.java diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/MapperExtrasPlugin.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/MapperExtrasPlugin.java index 9eae9839a1eb9..bac4ff473cb30 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/MapperExtrasPlugin.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/MapperExtrasPlugin.java @@ -36,10 +36,10 @@ public class MapperExtrasPlugin extends Plugin implements MapperPlugin, SearchPl public Map getMappers() { Map mappers = new LinkedHashMap<>(); mappers.put(ScaledFloatFieldMapper.CONTENT_TYPE, ScaledFloatFieldMapper.PARSER); - mappers.put(TokenCountFieldMapper.CONTENT_TYPE, new TokenCountFieldMapper.TypeParser()); - mappers.put(RankFeatureFieldMapper.CONTENT_TYPE, new RankFeatureFieldMapper.TypeParser()); - mappers.put(RankFeaturesFieldMapper.CONTENT_TYPE, new RankFeaturesFieldMapper.TypeParser()); - mappers.put(SearchAsYouTypeFieldMapper.CONTENT_TYPE, new SearchAsYouTypeFieldMapper.TypeParser()); + mappers.put(TokenCountFieldMapper.CONTENT_TYPE, TokenCountFieldMapper.PARSER); + mappers.put(RankFeatureFieldMapper.CONTENT_TYPE, RankFeatureFieldMapper.PARSER); + mappers.put(RankFeaturesFieldMapper.CONTENT_TYPE, RankFeaturesFieldMapper.PARSER); + mappers.put(SearchAsYouTypeFieldMapper.CONTENT_TYPE, SearchAsYouTypeFieldMapper.PARSER); return Collections.unmodifiableMap(mappers); } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java index 24c8c1b3b1615..0635f27e4c929 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java @@ -26,15 +26,13 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.elasticsearch.common.lucene.Lucene; -import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser.Token; -import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; -import java.util.Iterator; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.function.Supplier; @@ -42,7 +40,7 @@ /** * A {@link FieldMapper} that exposes Lucene's {@link FeatureField}. */ -public class RankFeatureFieldMapper extends FieldMapper { +public class RankFeatureFieldMapper extends ParametrizedFieldMapper { public static final String CONTENT_TYPE = "rank_feature"; @@ -57,43 +55,34 @@ public static class Defaults { } } - public static class Builder extends FieldMapper.Builder { + private static RankFeatureFieldType ft(FieldMapper in) { + return ((RankFeatureFieldMapper)in).fieldType(); + } + + public static class Builder extends ParametrizedFieldMapper.Builder { - private boolean positiveScoreImpact = true; + private final Parameter positiveScoreImpact + = Parameter.boolParam("positive_score_impact", false, m -> ft(m).positiveScoreImpact, true); + private final Parameter> meta = Parameter.metaParam(); public Builder(String name) { - super(name, Defaults.FIELD_TYPE); - builder = this; + super(name); } - public Builder positiveScoreImpact(boolean v) { - this.positiveScoreImpact = v; - return builder; + @Override + protected List> getParameters() { + return Arrays.asList(positiveScoreImpact, meta); } @Override public RankFeatureFieldMapper build(BuilderContext context) { - return new RankFeatureFieldMapper(name, fieldType, new RankFeatureFieldType(buildFullName(context), meta, positiveScoreImpact), - multiFieldsBuilder.build(this, context), copyTo, positiveScoreImpact); + return new RankFeatureFieldMapper(name, + new RankFeatureFieldType(buildFullName(context), meta.getValue(), positiveScoreImpact.getValue()), + multiFieldsBuilder.build(this, context), copyTo.build(), positiveScoreImpact.getValue()); } } - public static class TypeParser implements Mapper.TypeParser { - @Override - public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { - RankFeatureFieldMapper.Builder builder = new RankFeatureFieldMapper.Builder(name); - for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { - Map.Entry entry = iterator.next(); - String propName = entry.getKey(); - Object propNode = entry.getValue(); - if (propName.equals("positive_score_impact")) { - builder.positiveScoreImpact(XContentMapValues.nodeBooleanValue(propNode)); - iterator.remove(); - } - } - return builder; - } - } + public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n)); public static final class RankFeatureFieldType extends MappedFieldType { @@ -132,10 +121,9 @@ public Query termQuery(Object value, QueryShardContext context) { private final boolean positiveScoreImpact; - private RankFeatureFieldMapper(String simpleName, FieldType fieldType, MappedFieldType mappedFieldType, + private RankFeatureFieldMapper(String simpleName, MappedFieldType mappedFieldType, MultiFields multiFields, CopyTo copyTo, boolean positiveScoreImpact) { - super(simpleName, fieldType, mappedFieldType, multiFields, copyTo); - assert fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) <= 0; + super(simpleName, mappedFieldType, multiFields, copyTo); this.positiveScoreImpact = positiveScoreImpact; } @@ -201,23 +189,7 @@ protected String contentType() { } @Override - protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { - super.doXContentBody(builder, includeDefaults, params); - - if (includeDefaults || positiveScoreImpact == false) { - builder.field("positive_score_impact", positiveScoreImpact); - } - } - - @Override - protected boolean docValuesByDefault() { - return false; - } - - @Override - protected void mergeOptions(FieldMapper other, List conflicts) { - if (positiveScoreImpact != ((RankFeatureFieldMapper)other).positiveScoreImpact) { - conflicts.add("mapper [" + name() + "] has different [positive_score_impact] values"); - } + public ParametrizedFieldMapper.Builder getMergeBuilder() { + return new Builder(simpleName()).init(this); } } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java index 0a40d2a21d01a..7e11a6fa288d2 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java @@ -20,7 +20,6 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.document.FeatureField; -import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.search.Query; import org.elasticsearch.common.lucene.Lucene; @@ -30,6 +29,7 @@ import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.function.Supplier; @@ -38,43 +38,34 @@ * A {@link FieldMapper} that exposes Lucene's {@link FeatureField} as a sparse * vector of features. */ -public class RankFeaturesFieldMapper extends FieldMapper { +public class RankFeaturesFieldMapper extends ParametrizedFieldMapper { public static final String CONTENT_TYPE = "rank_features"; - public static class Defaults { - public static final FieldType FIELD_TYPE = new FieldType(); + public static class Builder extends ParametrizedFieldMapper.Builder { - static { - FIELD_TYPE.setTokenized(false); - FIELD_TYPE.setIndexOptions(IndexOptions.NONE); - FIELD_TYPE.setOmitNorms(true); - FIELD_TYPE.freeze(); - } - } - - public static class Builder extends FieldMapper.Builder { + private final Parameter> meta = Parameter.metaParam(); public Builder(String name) { - super(name, Defaults.FIELD_TYPE); + super(name); builder = this; } @Override - public RankFeaturesFieldMapper build(BuilderContext context) { - return new RankFeaturesFieldMapper( - name, fieldType, new RankFeaturesFieldType(buildFullName(context), meta), - multiFieldsBuilder.build(this, context), copyTo); + protected List> getParameters() { + return Collections.singletonList(meta); } - } - public static class TypeParser implements Mapper.TypeParser { @Override - public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { - return new Builder(name); + public RankFeaturesFieldMapper build(BuilderContext context) { + return new RankFeaturesFieldMapper( + name, new RankFeaturesFieldType(buildFullName(context), meta.getValue()), + multiFieldsBuilder.build(this, context), copyTo.build()); } } + public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n)); + public static final class RankFeaturesFieldType extends MappedFieldType { public RankFeaturesFieldType(String name, Map meta) { @@ -103,20 +94,20 @@ public Query termQuery(Object value, QueryShardContext context) { } } - private RankFeaturesFieldMapper(String simpleName, FieldType fieldType, MappedFieldType mappedFieldType, + private RankFeaturesFieldMapper(String simpleName, MappedFieldType mappedFieldType, MultiFields multiFields, CopyTo copyTo) { - super(simpleName, fieldType, mappedFieldType, multiFields, copyTo); + super(simpleName, mappedFieldType, multiFields, copyTo); assert fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) <= 0; } @Override - protected RankFeaturesFieldMapper clone() { - return (RankFeaturesFieldMapper) super.clone(); + public ParametrizedFieldMapper.Builder getMergeBuilder() { + return new Builder(simpleName()).init(this); } @Override - protected void mergeOptions(FieldMapper other, List conflicts) { - + protected RankFeaturesFieldMapper clone() { + return (RankFeaturesFieldMapper) super.clone(); } @Override @@ -157,7 +148,7 @@ public void parse(ParseContext context) throws IOException { } @Override - protected void parseCreateField(ParseContext context) throws IOException { + protected void parseCreateField(ParseContext context) { throw new AssertionError("parse is implemented directly"); } @@ -174,16 +165,6 @@ protected Object parseSourceValue(Object value) { }; } - @Override - protected boolean indexedByDefault() { - return false; - } - - @Override - protected boolean docValuesByDefault() { - return false; - } - @Override protected String contentType() { return CONTENT_TYPE; diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java index 3d99777905095..528ac40f669d6 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java @@ -47,12 +47,10 @@ import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.Operations; import org.elasticsearch.common.collect.Iterators; -import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.similarity.SimilarityProvider; -import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; @@ -63,11 +61,9 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Supplier; -import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeIntegerValue; import static org.elasticsearch.index.mapper.TextFieldMapper.TextFieldType.hasGaps; -import static org.elasticsearch.index.mapper.TypeParsers.checkNull; -import static org.elasticsearch.index.mapper.TypeParsers.parseTextField; /** * Mapper for a text field that optimizes itself for as-you-type completion by indexing its content into subfields. Each subfield @@ -83,7 +79,7 @@ * └── [ PrefixFieldMapper, PrefixFieldType, analysis wrapped with max_shingle_size-shingles and edge-ngrams ] * */ -public class SearchAsYouTypeFieldMapper extends FieldMapper { +public class SearchAsYouTypeFieldMapper extends ParametrizedFieldMapper { public static final String CONTENT_TYPE = "search_as_you_type"; private static final int MAX_SHINGLE_SIZE_LOWER_BOUND = 2; @@ -91,87 +87,84 @@ public class SearchAsYouTypeFieldMapper extends FieldMapper { private static final String PREFIX_FIELD_SUFFIX = "._index_prefix"; public static class Defaults { - public static final int MIN_GRAM = 1; public static final int MAX_GRAM = 20; public static final int MAX_SHINGLE_SIZE = 3; + } - public static final FieldType FIELD_TYPE = new FieldType(); + public static final TypeParser PARSER + = new TypeParser((n, c) -> new Builder(n, () -> c.getIndexAnalyzers().getDefaultIndexAnalyzer())); - static { - FIELD_TYPE.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); - FIELD_TYPE.freeze(); - } + private static SearchAsYouTypeFieldMapper toType(FieldMapper in) { + return (SearchAsYouTypeFieldMapper) in; } - public static class TypeParser implements Mapper.TypeParser { - - @Override - public Mapper.Builder parse(String name, - Map node, - ParserContext parserContext) throws MapperParsingException { - - final Builder builder = new Builder(name); - - builder.indexAnalyzer(parserContext.getIndexAnalyzers().getDefaultIndexAnalyzer()); - builder.searchAnalyzer(parserContext.getIndexAnalyzers().getDefaultSearchAnalyzer()); - builder.searchQuoteAnalyzer(parserContext.getIndexAnalyzers().getDefaultSearchQuoteAnalyzer()); - for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { - final Map.Entry entry = iterator.next(); - final String fieldName = entry.getKey(); - final Object fieldNode = entry.getValue(); - checkNull(fieldName, fieldNode); - if (fieldName.equals("max_shingle_size")) { - builder.maxShingleSize(nodeIntegerValue(fieldNode)); - iterator.remove(); - } else if (fieldName.equals("similarity")) { - SimilarityProvider similarityProvider = TypeParsers.resolveSimilarity(parserContext, fieldName, fieldNode.toString()); - builder.similarity(similarityProvider); - iterator.remove(); - } - // TODO should we allow to configure the prefix field - } - parseTextField(builder, name, node, parserContext); - return builder; - } + private static SearchAsYouTypeFieldType ft(FieldMapper in) { + return toType(in).fieldType(); } - public static class Builder extends FieldMapper.Builder { - private int maxShingleSize = Defaults.MAX_SHINGLE_SIZE; - private SimilarityProvider similarity; + public static class Builder extends ParametrizedFieldMapper.Builder { - public Builder(String name) { - super(name, Defaults.FIELD_TYPE); - this.builder = this; - } + private final Parameter index = Parameter.indexParam(m -> toType(m).index, true); + private final Parameter store = Parameter.storeParam(m -> toType(m).store, false); - public void similarity(SimilarityProvider similarity) { - this.similarity = similarity; - } + // This is only here because for some reason the initial impl of this always serialized + // `doc_values=false`, even though it cannot be set; and so we need to continue + // serializing it forever because of mapper assertions in mixed clusters. + private final Parameter docValues = Parameter.docValuesParam(m -> false, false) + .setValidator(v -> { + if (v) { + throw new MapperParsingException("Cannot set [doc_values] on field of type [search_as_you_type]"); + } + }) + .alwaysSerialize(); + + private final Parameter maxShingleSize = Parameter.intParam("max_shingle_size", false, + m -> toType(m).maxShingleSize, Defaults.MAX_SHINGLE_SIZE) + .setValidator(v -> { + if (v < MAX_SHINGLE_SIZE_LOWER_BOUND || v > MAX_SHINGLE_SIZE_UPPER_BOUND) { + throw new MapperParsingException("[max_shingle_size] must be at least [" + MAX_SHINGLE_SIZE_LOWER_BOUND + + "] and at most " + "[" + MAX_SHINGLE_SIZE_UPPER_BOUND + "], got [" + v + "]"); + } + }) + .alwaysSerialize(); - public Builder maxShingleSize(int maxShingleSize) { - if (maxShingleSize < MAX_SHINGLE_SIZE_LOWER_BOUND || maxShingleSize > MAX_SHINGLE_SIZE_UPPER_BOUND) { - throw new MapperParsingException("[max_shingle_size] must be at least [" + MAX_SHINGLE_SIZE_LOWER_BOUND + "] and at most " + - "[" + MAX_SHINGLE_SIZE_UPPER_BOUND + "], got [" + maxShingleSize + "]"); - } - this.maxShingleSize = maxShingleSize; - return builder; + final TextParams.Analyzers analyzers; + final Parameter similarity = TextParams.similarity(m -> ft(m).getTextSearchInfo().getSimilarity()); + + final Parameter indexOptions = TextParams.indexOptions(m -> toType(m).indexOptions); + final Parameter norms = TextParams.norms(true, m -> ft(m).getTextSearchInfo().hasNorms()); + final Parameter termVectors = TextParams.termVectors(m -> toType(m).termVectors); + + private final Parameter> meta = Parameter.metaParam(); + + public Builder(String name, Supplier defaultAnalyzer) { + super(name); + this.analyzers = new TextParams.Analyzers(defaultAnalyzer); } @Override - public Builder docValues(boolean docValues) { - if (docValues) { - throw new IllegalArgumentException("mapper [" + name() + "] of type [search_as_you_type] does not support doc values"); - } - return this; + protected List> getParameters() { + return List.of(index, store, docValues, maxShingleSize, + analyzers.indexAnalyzer, analyzers.searchAnalyzer, analyzers.searchQuoteAnalyzer, similarity, + indexOptions, norms, termVectors, meta); } @Override public SearchAsYouTypeFieldMapper build(Mapper.BuilderContext context) { - SearchAsYouTypeFieldType ft = new SearchAsYouTypeFieldType(buildFullName(context), fieldType, similarity, - searchAnalyzer, searchQuoteAnalyzer, meta); - ft.setIndexAnalyzer(indexAnalyzer); + FieldType fieldType = new FieldType(); + fieldType.setIndexOptions(TextParams.toIndexOptions(index.getValue(), indexOptions.getValue())); + fieldType.setOmitNorms(norms.getValue() == false); + fieldType.setStored(store.getValue()); + TextParams.setTermVectorParams(termVectors.getValue(), fieldType); + + NamedAnalyzer indexAnalyzer = analyzers.getIndexAnalyzer(); + NamedAnalyzer searchAnalyzer = analyzers.getSearchAnalyzer(); + + SearchAsYouTypeFieldType ft = new SearchAsYouTypeFieldType(buildFullName(context), fieldType, similarity.getValue(), + analyzers.getSearchAnalyzer(), analyzers.getSearchQuoteAnalyzer(), meta.getValue()); + ft.setIndexAnalyzer(analyzers.getIndexAnalyzer()); // set up the prefix field FieldType prefixft = new FieldType(fieldType); @@ -181,21 +174,20 @@ public SearchAsYouTypeFieldMapper build(Mapper.BuilderContext context) { final String fullName = buildFullName(context); // wrap the root field's index analyzer with shingles and edge ngrams final Analyzer prefixIndexWrapper = - SearchAsYouTypeAnalyzer.withShingleAndPrefix(indexAnalyzer.analyzer(), maxShingleSize); + SearchAsYouTypeAnalyzer.withShingleAndPrefix(indexAnalyzer.analyzer(), maxShingleSize.getValue()); // wrap the root field's search analyzer with only shingles final NamedAnalyzer prefixSearchWrapper = new NamedAnalyzer(searchAnalyzer.name(), searchAnalyzer.scope(), - SearchAsYouTypeAnalyzer.withShingle(searchAnalyzer.analyzer(), maxShingleSize)); + SearchAsYouTypeAnalyzer.withShingle(searchAnalyzer.analyzer(), maxShingleSize.getValue())); // don't wrap the root field's search quote analyzer as prefix field doesn't support phrase queries - TextSearchInfo prefixSearchInfo = new TextSearchInfo(prefixft, similarity, prefixSearchWrapper, searchQuoteAnalyzer); + TextSearchInfo prefixSearchInfo = new TextSearchInfo(prefixft, similarity.getValue(), prefixSearchWrapper, searchAnalyzer); final PrefixFieldType prefixFieldType = new PrefixFieldType(fullName, prefixSearchInfo, Defaults.MIN_GRAM, Defaults.MAX_GRAM); prefixFieldType.setIndexAnalyzer(new NamedAnalyzer(indexAnalyzer.name(), AnalyzerScope.INDEX, prefixIndexWrapper)); final PrefixFieldMapper prefixFieldMapper = new PrefixFieldMapper(prefixft, prefixFieldType); - // set up the shingle fields - final ShingleFieldMapper[] shingleFieldMappers = new ShingleFieldMapper[maxShingleSize - 1]; - final ShingleFieldType[] shingleFieldTypes = new ShingleFieldType[maxShingleSize - 1]; + final ShingleFieldMapper[] shingleFieldMappers = new ShingleFieldMapper[maxShingleSize.getValue() - 1]; + final ShingleFieldType[] shingleFieldTypes = new ShingleFieldType[maxShingleSize.getValue() - 1]; for (int i = 0; i < shingleFieldMappers.length; i++) { final int shingleSize = i + 2; FieldType shingleft = new FieldType(fieldType); @@ -206,10 +198,10 @@ public SearchAsYouTypeFieldMapper build(Mapper.BuilderContext context) { SearchAsYouTypeAnalyzer.withShingle(indexAnalyzer.analyzer(), shingleSize); final NamedAnalyzer shingleSearchWrapper = new NamedAnalyzer(searchAnalyzer.name(), searchAnalyzer.scope(), SearchAsYouTypeAnalyzer.withShingle(searchAnalyzer.analyzer(), shingleSize)); - final NamedAnalyzer shingleSearchQuoteWrapper = new NamedAnalyzer(searchQuoteAnalyzer.name(), searchQuoteAnalyzer.scope(), - SearchAsYouTypeAnalyzer.withShingle(searchQuoteAnalyzer.analyzer(), shingleSize)); + final NamedAnalyzer shingleSearchQuoteWrapper = new NamedAnalyzer(searchAnalyzer.name(), searchAnalyzer.scope(), + SearchAsYouTypeAnalyzer.withShingle(searchAnalyzer.analyzer(), shingleSize)); TextSearchInfo textSearchInfo - = new TextSearchInfo(shingleft, similarity, shingleSearchWrapper, shingleSearchQuoteWrapper); + = new TextSearchInfo(shingleft, similarity.getValue(), shingleSearchWrapper, shingleSearchQuoteWrapper); final ShingleFieldType shingleFieldType = new ShingleFieldType(fieldName, shingleSize, textSearchInfo); shingleFieldType.setIndexAnalyzer(new NamedAnalyzer(indexAnalyzer.name(), AnalyzerScope.INDEX, shingleIndexWrapper)); shingleFieldType.setPrefixFieldType(prefixFieldType); @@ -218,8 +210,7 @@ public SearchAsYouTypeFieldMapper build(Mapper.BuilderContext context) { } ft.setPrefixField(prefixFieldType); ft.setShingleFields(shingleFieldTypes); - return new SearchAsYouTypeFieldMapper(name, fieldType, ft, copyTo, - maxShingleSize, prefixFieldMapper, shingleFieldMappers); + return new SearchAsYouTypeFieldMapper(name, ft, copyTo.build(), prefixFieldMapper, shingleFieldMappers, this); } } @@ -243,6 +234,7 @@ private static int countPosition(TokenStream stream) throws IOException { */ static class SearchAsYouTypeFieldType extends StringFieldType { + final FieldType fieldType; PrefixFieldType prefixField; ShingleFieldType[] shingleFields = new ShingleFieldType[0]; @@ -250,6 +242,7 @@ static class SearchAsYouTypeFieldType extends StringFieldType { NamedAnalyzer searchAnalyzer, NamedAnalyzer searchQuoteAnalyzer, Map meta) { super(name, fieldType.indexOptions() != IndexOptions.NONE, fieldType.stored(), false, new TextSearchInfo(fieldType, similarity, searchAnalyzer, searchQuoteAnalyzer), meta); + this.fieldType = fieldType; } public void setPrefixField(PrefixFieldType prefixField) { @@ -538,21 +531,29 @@ public SpanQuery spanPrefixQuery(String value, SpanMultiTermQueryWrapper.SpanRew } } + private final boolean index; + private final boolean store; + private final String indexOptions; + private final String termVectors; + private final int maxShingleSize; - private PrefixFieldMapper prefixField; + private final PrefixFieldMapper prefixField; private final ShingleFieldMapper[] shingleFields; public SearchAsYouTypeFieldMapper(String simpleName, - FieldType fieldType, SearchAsYouTypeFieldType mappedFieldType, CopyTo copyTo, - int maxShingleSize, PrefixFieldMapper prefixField, - ShingleFieldMapper[] shingleFields) { - super(simpleName, fieldType, mappedFieldType, MultiFields.empty(), copyTo); + ShingleFieldMapper[] shingleFields, + Builder builder) { + super(simpleName, mappedFieldType, MultiFields.empty(), copyTo); this.prefixField = prefixField; this.shingleFields = shingleFields; - this.maxShingleSize = maxShingleSize; + this.maxShingleSize = builder.maxShingleSize.getValue(); + this.index = builder.index.getValue(); + this.store = builder.store.getValue(); + this.indexOptions = builder.indexOptions.getValue(); + this.termVectors = builder.termVectors.getValue(); } @Override @@ -562,12 +563,12 @@ protected void parseCreateField(ParseContext context) throws IOException { return; } - context.doc().add(new Field(fieldType().name(), value, fieldType)); + context.doc().add(new Field(fieldType().name(), value, fieldType().fieldType)); for (ShingleFieldMapper subFieldMapper : shingleFields) { context.doc().add(new Field(subFieldMapper.fieldType().name(), value, subFieldMapper.getLuceneFieldType())); } context.doc().add(new Field(prefixField.fieldType().name(), value, prefixField.getLuceneFieldType())); - if (fieldType.omitNorms()) { + if (fieldType().fieldType.omitNorms()) { createFieldNamesField(context); } } @@ -583,20 +584,8 @@ protected String contentType() { } @Override - protected void mergeOptions(FieldMapper other, List conflicts) { - final SearchAsYouTypeFieldMapper m = (SearchAsYouTypeFieldMapper) other; - if (this.shingleFields.length != m.shingleFields.length) { - conflicts.add("mapper [" + name() + "] has a different [max_shingle_size]"); - } else { - this.prefixField = (PrefixFieldMapper) this.prefixField.merge(m.prefixField); - for (int i = 0; i < m.shingleFields.length; i++) { - this.shingleFields[i] = (ShingleFieldMapper) this.shingleFields[i].merge(m.shingleFields[i]); - } - } - if (Objects.equals(this.fieldType().getTextSearchInfo().getSimilarity(), - other.fieldType().getTextSearchInfo().getSimilarity()) == false) { - conflicts.add("mapper [" + name() + "] has different [similarity] settings"); - } + public ParametrizedFieldMapper.Builder getMergeBuilder() { + return new Builder(simpleName(), () -> fieldType().indexAnalyzer()).init(this); } public static String getShingleFieldName(String parentField, int shingleSize) { @@ -620,18 +609,6 @@ public ShingleFieldMapper[] shingleFields() { return shingleFields; } - @Override - protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { - super.doXContentBody(builder, includeDefaults, params); - doXContentAnalyzers(builder, includeDefaults); - if (fieldType().getTextSearchInfo().getSimilarity() != null) { - builder.field("similarity", fieldType().getTextSearchInfo().getSimilarity().name()); - } else if (includeDefaults) { - builder.field("similarity", SimilarityService.DEFAULT_SIMILARITY); - } - builder.field("max_shingle_size", maxShingleSize); - } - @Override public Iterator iterator() { List subIterators = new ArrayList<>(); diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java index 087b828cdb232..e8b6cf8d00b39 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java @@ -22,115 +22,86 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; -import org.apache.lucene.document.FieldType; -import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; -import java.util.Iterator; +import java.util.Arrays; import java.util.List; import java.util.Map; -import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue; import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeIntegerValue; -import static org.elasticsearch.index.mapper.TypeParsers.parseField; /** * A {@link FieldMapper} that takes a string and writes a count of the tokens in that string * to the index. In most ways the mapper acts just like an {@link NumberFieldMapper}. */ -public class TokenCountFieldMapper extends FieldMapper { +public class TokenCountFieldMapper extends ParametrizedFieldMapper { public static final String CONTENT_TYPE = "token_count"; - public static class Defaults { - public static final boolean DEFAULT_POSITION_INCREMENTS = true; + private static TokenCountFieldMapper toType(FieldMapper in) { + return (TokenCountFieldMapper) in; } - public static class Builder extends FieldMapper.Builder { - private NamedAnalyzer analyzer; - private Integer nullValue; - private boolean enablePositionIncrements = Defaults.DEFAULT_POSITION_INCREMENTS; + public static class Builder extends ParametrizedFieldMapper.Builder { - public Builder(String name) { - super(name, new FieldType()); - builder = this; - } - - public Builder analyzer(NamedAnalyzer analyzer) { - this.analyzer = analyzer; - return this; - } + private final Parameter index = Parameter.indexParam(m -> toType(m).index, true); + private final Parameter hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true); + private final Parameter store = Parameter.storeParam(m -> toType(m).store, false); - public NamedAnalyzer analyzer() { - return analyzer; - } + private final Parameter analyzer + = Parameter.analyzerParam("analyzer", true, m -> toType(m).analyzer, () -> null); + private final Parameter nullValue = new Parameter<>( + "null_value", false, () -> null, + (n, c, o) -> o == null ? null : nodeIntegerValue(o), m -> toType(m).nullValue).acceptsNull(); + private final Parameter enablePositionIncrements + = Parameter.boolParam("enable_position_increments", false, m -> toType(m).enablePositionIncrements, true); - public Builder enablePositionIncrements(boolean enablePositionIncrements) { - this.enablePositionIncrements = enablePositionIncrements; - return this; - } + private final Parameter> meta = Parameter.metaParam(); - public boolean enablePositionIncrements() { - return enablePositionIncrements; - } - - public Builder nullValue(Integer nullValue) { - this.nullValue = nullValue; - return this; + public Builder(String name) { + super(name); } @Override - public TokenCountFieldMapper build(BuilderContext context) { - return new TokenCountFieldMapper(name, fieldType, - new NumberFieldMapper.NumberFieldType(buildFullName(context), NumberFieldMapper.NumberType.INTEGER), - analyzer, enablePositionIncrements, nullValue, - multiFieldsBuilder.build(this, context), copyTo); + protected List> getParameters() { + return Arrays.asList(index, hasDocValues, store, analyzer, nullValue, enablePositionIncrements, meta); } - } - public static class TypeParser implements Mapper.TypeParser { @Override - public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { - TokenCountFieldMapper.Builder builder = new TokenCountFieldMapper.Builder(name); - for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { - Map.Entry entry = iterator.next(); - String propName = entry.getKey(); - Object propNode = entry.getValue(); - if (propName.equals("null_value")) { - builder.nullValue(nodeIntegerValue(propNode)); - iterator.remove(); - } else if (propName.equals("analyzer")) { - NamedAnalyzer analyzer = parserContext.getIndexAnalyzers().get(propNode.toString()); - if (analyzer == null) { - throw new MapperParsingException("Analyzer [" + propNode.toString() + "] not found for field [" + name + "]"); - } - builder.analyzer(analyzer); - iterator.remove(); - } else if (propName.equals("enable_position_increments")) { - builder.enablePositionIncrements(nodeBooleanValue(propNode)); - iterator.remove(); - } - } - parseField(builder, name, node, parserContext); - if (builder.analyzer() == null) { + public TokenCountFieldMapper build(BuilderContext context) { + if (analyzer.getValue() == null) { throw new MapperParsingException("Analyzer must be set for field [" + name + "] but wasn't."); } - return builder; + MappedFieldType ft = new NumberFieldMapper.NumberFieldType( + buildFullName(context), + NumberFieldMapper.NumberType.INTEGER, + index.getValue(), + store.getValue(), + hasDocValues.getValue(), + meta.getValue()); + return new TokenCountFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), this); } } - private NamedAnalyzer analyzer; + public static TypeParser PARSER = new TypeParser((n, c) -> new Builder(n)); + + private final boolean index; + private final boolean hasDocValues; + private final boolean store; + private final NamedAnalyzer analyzer; private final boolean enablePositionIncrements; - private Integer nullValue; - - protected TokenCountFieldMapper(String simpleName, FieldType fieldType, MappedFieldType defaultFieldType, - NamedAnalyzer analyzer, boolean enablePositionIncrements, Integer nullValue, - MultiFields multiFields, CopyTo copyTo) { - super(simpleName, fieldType, defaultFieldType, multiFields, copyTo); - this.analyzer = analyzer; - this.enablePositionIncrements = enablePositionIncrements; - this.nullValue = nullValue; + private final Integer nullValue; + + protected TokenCountFieldMapper(String simpleName, MappedFieldType defaultFieldType, + MultiFields multiFields, CopyTo copyTo, Builder builder) { + super(simpleName, defaultFieldType, multiFields, copyTo); + this.analyzer = builder.analyzer.getValue(); + this.enablePositionIncrements = builder.enablePositionIncrements.getValue(); + this.nullValue = builder.nullValue.getValue(); + this.index = builder.index.getValue(); + this.hasDocValues = builder.hasDocValues.getValue(); + this.store = builder.store.getValue(); } @Override @@ -153,10 +124,9 @@ protected void parseCreateField(ParseContext context) throws IOException { tokenCount = countPositions(analyzer, name(), value, enablePositionIncrements); } - boolean indexed = fieldType().isSearchable(); - boolean docValued = fieldType().hasDocValues(); - boolean stored = fieldType.stored(); - context.doc().addAll(NumberFieldMapper.NumberType.INTEGER.createFields(fieldType().name(), tokenCount, indexed, docValued, stored)); + context.doc().addAll( + NumberFieldMapper.NumberType.INTEGER.createFields(fieldType().name(), tokenCount, index, hasDocValues, store) + ); } @Override @@ -224,21 +194,7 @@ protected String contentType() { } @Override - protected void mergeOptions(FieldMapper other, List conflicts) { - // TODO we should ban updating analyzers and null values as well - if (this.enablePositionIncrements != ((TokenCountFieldMapper)other).enablePositionIncrements) { - conflicts.add("mapper [" + name() + "] has a different [enable_position_increments] setting"); - } - this.analyzer = ((TokenCountFieldMapper)other).analyzer; - } - - @Override - protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { - super.doXContentBody(builder, includeDefaults, params); - builder.field("analyzer", analyzer()); - if (includeDefaults || enablePositionIncrements() != Defaults.DEFAULT_POSITION_INCREMENTS) { - builder.field("enable_position_increments", enablePositionIncrements()); - } + public ParametrizedFieldMapper.Builder getMergeBuilder() { + return new Builder(simpleName()).init(this); } - } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java index aa8b4eed03b03..63c6fa49062cb 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java @@ -31,28 +31,21 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.plugins.Plugin; -import org.hamcrest.Matchers; import java.io.IOException; import java.util.Arrays; import java.util.Collection; import java.util.List; -import java.util.Set; import static org.hamcrest.Matchers.instanceOf; -public class RankFeatureFieldMapperTests extends FieldMapperTestCase2 { +public class RankFeatureFieldMapperTests extends MapperTestCase { @Override protected void writeFieldValue(XContentBuilder builder) throws IOException { builder.value(10); } - @Override - protected Set unsupportedProperties() { - return Set.of("analyzer", "similarity", "store", "doc_values", "index"); - } - @Override protected void registerParameters(ParameterChecker checker) throws IOException { checker.registerConflictCheck("positive_score_impact", b -> b.field("positive_score_impact", false)); @@ -81,21 +74,11 @@ static int getFrequency(TokenStream tk) throws IOException { return freq; } - @Override - protected RankFeatureFieldMapper.Builder newBuilder() { - return new RankFeatureFieldMapper.Builder("rank-feature"); - } - @Override protected void minimalMapping(XContentBuilder b) throws IOException { b.field("type", "rank_feature"); } - @Override - protected boolean supportsMeta() { - return false; - } - public void testDefaults() throws Exception { DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); assertEquals(Strings.toString(fieldMapping(this::minimalMapping)), mapper.mappingSource().toString()); @@ -103,7 +86,7 @@ public void testDefaults() throws Exception { ParsedDocument doc1 = mapper.parse(source(b -> b.field("field", 10))); IndexableField[] fields = doc1.rootDoc().getFields("_feature"); assertEquals(1, fields.length); - assertThat(fields[0], Matchers.instanceOf(FeatureField.class)); + assertThat(fields[0], instanceOf(FeatureField.class)); FeatureField featureField1 = (FeatureField) fields[0]; ParsedDocument doc2 = mapper.parse(source(b -> b.field("field", 12))); @@ -122,7 +105,7 @@ public void testNegativeScoreImpact() throws Exception { ParsedDocument doc1 = mapper.parse(source(b -> b.field("field", 10))); IndexableField[] fields = doc1.rootDoc().getFields("_feature"); assertEquals(1, fields.length); - assertThat(fields[0], Matchers.instanceOf(FeatureField.class)); + assertThat(fields[0], instanceOf(FeatureField.class)); FeatureField featureField1 = (FeatureField) fields[0]; ParsedDocument doc2 = mapper.parse(source(b -> b.field("field", 12))); diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapperTests.java index 5354c8758e478..3eb4853b87c18 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapperTests.java @@ -30,9 +30,8 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; -import java.util.Set; -public class RankFeaturesFieldMapperTests extends FieldMapperTestCase2 { +public class RankFeaturesFieldMapperTests extends MapperTestCase { @Override protected void writeFieldValue(XContentBuilder builder) throws IOException { @@ -45,11 +44,6 @@ protected void assertExistsQuery(MapperService mapperService) { assertEquals("[rank_features] fields do not support [exists] queries", iae.getMessage()); } - @Override - protected Set unsupportedProperties() { - return Set.of("analyzer", "similarity", "store", "doc_values", "index"); - } - @Override protected Collection getPlugins() { return List.of(new MapperExtrasPlugin()); @@ -117,9 +111,4 @@ public void testRejectMultiValuedFields() throws MapperParsingException, IOExcep assertEquals("[rank_features] fields do not support indexing multiple values for the same rank feature [foo.field.bar] in " + "the same document", e.getCause().getMessage()); } - - @Override - protected RankFeaturesFieldMapper.Builder newBuilder() { - return new RankFeaturesFieldMapper.Builder("rf"); - } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java index 1ee86eec89184..7a7bdc42bc992 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java @@ -74,7 +74,7 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.core.IsInstanceOf.instanceOf; -public class SearchAsYouTypeFieldMapperTests extends FieldMapperTestCase2 { +public class SearchAsYouTypeFieldMapperTests extends MapperTestCase { @Override protected void registerParameters(ParameterChecker checker) throws IOException { @@ -126,24 +126,11 @@ protected void writeFieldValue(XContentBuilder builder) throws IOException { builder.value("new york city"); } - @Override - protected Set unsupportedProperties() { - return Set.of("doc_values"); - } - @Override protected Collection getPlugins() { return List.of(new MapperExtrasPlugin()); } - @Override - protected SearchAsYouTypeFieldMapper.Builder newBuilder() { - return new SearchAsYouTypeFieldMapper.Builder("sayt") - .indexAnalyzer(new NamedAnalyzer("standard", AnalyzerScope.INDEX, new StandardAnalyzer())) - .searchAnalyzer(new NamedAnalyzer("standard", AnalyzerScope.INDEX, new StandardAnalyzer())) - .searchQuoteAnalyzer(new NamedAnalyzer("standard", AnalyzerScope.INDEX, new StandardAnalyzer())); - } - @Override protected IndexAnalyzers createIndexAnalyzers(IndexSettings indexSettings) { NamedAnalyzer dflt = new NamedAnalyzer( @@ -196,7 +183,7 @@ public void testDefaultConfiguration() throws IOException { getShingleFieldMapper(defaultMapper, "field._2gram").fieldType(), 2, "default", prefixFieldMapper.fieldType()); assertShingleFieldType( getShingleFieldMapper(defaultMapper, "field._3gram").fieldType(), 3, "default", prefixFieldMapper.fieldType()); - } + } public void testConfiguration() throws IOException { int maxShingleSize = 4; @@ -239,7 +226,7 @@ public void testSimpleMerge() throws IOException { b.endObject(); b.startObject("b_field").field("type", "text").endObject(); }))); - assertThat(e.getMessage(), containsString("different [max_shingle_size]")); + assertThat(e.getMessage(), containsString("Cannot update parameter [max_shingle_size]")); } public void testMultiFields() throws IOException { @@ -271,7 +258,7 @@ private void assertMultiField(int shingleSize) throws IOException { assertThat(fieldType, instanceOf(ShingleFieldType.class)); ShingleFieldType ft = (ShingleFieldType) fieldType; assertEquals(i, ft.shingleSize); - assertTrue(prefixFieldType == ft.prefixFieldType); + assertSame(prefixFieldType, ft.prefixFieldType); } ParsedDocument doc = mapperService.documentMapper().parse(source(b -> b.field("field", "new york city"))); @@ -287,8 +274,10 @@ public void testIndexOptions() throws IOException { fieldMapping(b -> b.field("type", "search_as_you_type").field("index_options", "offsets")) ); + assertThat(getRootFieldMapper(mapper, "field").fieldType().fieldType.indexOptions(), + equalTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)); + Stream.of( - getRootFieldMapper(mapper, "field"), getPrefixFieldMapper(mapper, "field._index_prefix"), getShingleFieldMapper(mapper, "field._2gram"), getShingleFieldMapper(mapper, "field._3gram") @@ -299,7 +288,7 @@ public void testIndexOptions() throws IOException { public void testStore() throws IOException { DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "search_as_you_type").field("store", true))); - assertTrue(getRootFieldMapper(mapper, "field").fieldType.stored()); + assertTrue(getRootFieldMapper(mapper, "field").fieldType().fieldType.stored()); Stream.of( getPrefixFieldMapper(mapper, "field._index_prefix"), getShingleFieldMapper(mapper, "field._2gram"), @@ -321,8 +310,9 @@ public void testIndex() throws IOException { public void testTermVectors() throws IOException { DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "search_as_you_type").field("term_vector", "yes"))); + assertTrue(getRootFieldMapper(mapper, "field").fieldType().fieldType.storeTermVectors()); + Stream.of( - getRootFieldMapper(mapper, "field"), getShingleFieldMapper(mapper, "field._2gram"), getShingleFieldMapper(mapper, "field._3gram") ).forEach(m -> assertTrue("for " + m.name(), m.fieldType.storeTermVectors())); @@ -350,8 +340,9 @@ public void testNorms() throws IOException { { DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "search_as_you_type").field("norms", false))); + assertTrue(getRootFieldMapper(mapper, "field").fieldType().fieldType.omitNorms()); + Stream.of( - getRootFieldMapper(mapper, "field"), getPrefixFieldMapper(mapper, "field._index_prefix"), getShingleFieldMapper(mapper, "field._2gram"), getShingleFieldMapper(mapper, "field._3gram") @@ -359,7 +350,6 @@ public void testNorms() throws IOException { } } - public void testDocumentParsingSingleValue() throws IOException { documentParsingTestCase(Collections.singleton(randomAlphaOfLengthBetween(5, 20))); } @@ -577,7 +567,7 @@ private void documentParsingTestCase(Collection values) throws IOExcepti IndexableField[] prefixFields = parsedDocument.rootDoc().getFields("field._index_prefix"); IndexableField[] shingle2Fields = parsedDocument.rootDoc().getFields("field._2gram"); IndexableField[] shingle3Fields = parsedDocument.rootDoc().getFields("field._3gram"); - for (IndexableField[] fields : new IndexableField[][]{ rootFields, prefixFields, shingle2Fields, shingle3Fields}) { + for (IndexableField[] fields : new IndexableField[][]{rootFields, prefixFields, shingle2Fields, shingle3Fields}) { Set expectedValues = Arrays.stream(fields).map(IndexableField::stringValue).collect(Collectors.toSet()); assertThat(values, equalTo(expectedValues)); } @@ -610,7 +600,7 @@ private static void assertSearchAsYouTypeFieldType(SearchAsYouTypeFieldType fiel String analyzerName, PrefixFieldType prefixFieldType) { - assertThat(fieldType.shingleFields.length, equalTo(maxShingleSize-1)); + assertThat(fieldType.shingleFields.length, equalTo(maxShingleSize - 1)); for (NamedAnalyzer analyzer : asList(fieldType.indexAnalyzer(), fieldType.getTextSearchInfo().getSearchAnalyzer())) { assertThat(analyzer.name(), equalTo(analyzerName)); } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldTypeTests.java index bc84c370df42b..de5cdaa2a36cd 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldTypeTests.java @@ -49,8 +49,14 @@ public class SearchAsYouTypeFieldTypeTests extends FieldTypeTestCase { UNSEARCHABLE.freeze(); } + private static final FieldType SEARCHABLE = new FieldType(); + static { + SEARCHABLE.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); + SEARCHABLE.freeze(); + } + private static SearchAsYouTypeFieldType createFieldType() { - final SearchAsYouTypeFieldType fieldType = new SearchAsYouTypeFieldType(NAME, Defaults.FIELD_TYPE, null, + final SearchAsYouTypeFieldType fieldType = new SearchAsYouTypeFieldType(NAME, SEARCHABLE, null, Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER, Collections.emptyMap()); fieldType.setPrefixField(new PrefixFieldType(NAME, TextSearchInfo.SIMPLE_MATCH_ONLY, Defaults.MIN_GRAM, Defaults.MAX_GRAM)); fieldType.setShingleFields(new ShingleFieldType[] { diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/TokenCountFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/TokenCountFieldMapperTests.java index c032abe5d42a1..d7df1c1f08ede 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/TokenCountFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/TokenCountFieldMapperTests.java @@ -24,64 +24,73 @@ import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenStream; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.compress.CompressedXContent; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.index.IndexService; +import org.apache.lucene.analysis.core.KeywordAnalyzer; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.analysis.AnalyzerScope; +import org.elasticsearch.index.analysis.IndexAnalyzers; +import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.test.ESSingleNodeTestCase; -import org.elasticsearch.test.InternalSettingsPlugin; import java.io.IOException; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.Map; -import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; /** * Test for {@link TokenCountFieldMapper}. */ -public class TokenCountFieldMapperTests extends ESSingleNodeTestCase { +public class TokenCountFieldMapperTests extends MapperTestCase { @Override - protected Collection> getPlugins() { - return pluginList(InternalSettingsPlugin.class, MapperExtrasPlugin.class); + protected Collection getPlugins() { + return Collections.singletonList(new MapperExtrasPlugin()); } - public void testMerge() throws IOException { - String stage1Mapping = Strings.toString(XContentFactory.jsonBuilder().startObject() - .startObject("person") - .startObject("properties") - .startObject("tc") - .field("type", "token_count") - .field("analyzer", "keyword") - .endObject() - .endObject() - .endObject().endObject()); - MapperService mapperService = createIndex("test").mapperService(); - DocumentMapper stage1 = mapperService.merge("person", - new CompressedXContent(stage1Mapping), MapperService.MergeReason.MAPPING_UPDATE); - - String stage2Mapping = Strings.toString(XContentFactory.jsonBuilder().startObject() - .startObject("person") - .startObject("properties") - .startObject("tc") - .field("type", "token_count") - .field("analyzer", "standard") - .endObject() - .endObject() - .endObject().endObject()); - DocumentMapper stage2 = mapperService.merge("person", - new CompressedXContent(stage2Mapping), MapperService.MergeReason.MAPPING_UPDATE); - - // previous mapper has not been modified - assertThat(((TokenCountFieldMapper) stage1.mappers().getMapper("tc")).analyzer(), equalTo("keyword")); - // but the new one has the change - assertThat(((TokenCountFieldMapper) stage2.mappers().getMapper("tc")).analyzer(), equalTo("standard")); + @Override + protected void minimalMapping(XContentBuilder b) throws IOException { + b.field("type", "token_count").field("analyzer", "keyword"); + } + + @Override + protected void writeFieldValue(XContentBuilder builder) throws IOException { + builder.value("some words"); + } + + @Override + protected void registerParameters(ParameterChecker checker) throws IOException { + checker.registerConflictCheck("index", b -> b.field("index", false)); + checker.registerConflictCheck("store", b -> b.field("store", true)); + checker.registerConflictCheck("doc_values", b -> b.field("doc_values", false)); + checker.registerConflictCheck("null_value", b -> b.field("null_value", 1)); + checker.registerConflictCheck("enable_position_increments", b -> b.field("enable_position_increments", false)); + checker.registerUpdateCheck( + this::minimalMapping, + b -> b.field("type", "token_count").field("analyzer", "standard"), + m -> { + TokenCountFieldMapper tcfm = (TokenCountFieldMapper) m; + assertThat(tcfm.analyzer(), equalTo("standard")); + }); + } + + @Override + protected IndexAnalyzers createIndexAnalyzers(IndexSettings indexSettings) { + NamedAnalyzer dflt = new NamedAnalyzer( + "default", + AnalyzerScope.INDEX, + new StandardAnalyzer() + ); + NamedAnalyzer standard = new NamedAnalyzer("standard", AnalyzerScope.INDEX, new StandardAnalyzer()); + NamedAnalyzer keyword = new NamedAnalyzer("keyword", AnalyzerScope.INDEX, new KeywordAnalyzer()); + return new IndexAnalyzers( + Map.of("default", dflt, "standard", standard, "keyword", keyword), + Map.of(), + Map.of() + ); } /** @@ -120,33 +129,12 @@ private Analyzer createMockAnalyzer() { Collections.shuffle(Arrays.asList(tokens), random()); final TokenStream tokenStream = new CannedTokenStream(finalTokenIncrement, 0, tokens); // TODO: we have no CannedAnalyzer? - Analyzer analyzer = new Analyzer() { + return new Analyzer() { @Override public TokenStreamComponents createComponents(String fieldName) { return new TokenStreamComponents(new MockTokenizer(), tokenStream); } }; - return analyzer; - } - - public void testEmptyName() throws IOException { - IndexService indexService = createIndex("test"); - DocumentMapperParser parser = indexService.mapperService().documentMapperParser(); - String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject() - .startObject("type") - .startObject("properties") - .startObject("") - .field("type", "token_count") - .field("analyzer", "standard") - .endObject() - .endObject() - .endObject().endObject()); - - // Empty name not allowed in index created after 5.0 - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> parser.parse("type", new CompressedXContent(mapping)) - ); - assertThat(e.getMessage(), containsString("name cannot be empty string")); } public void testParseNullValue() throws Exception { @@ -168,31 +156,27 @@ public void testParseNotNullValue() throws Exception { } private DocumentMapper createIndexWithTokenCountField() throws IOException { - final String content = Strings.toString(XContentFactory.jsonBuilder().startObject() - .startObject("person") - .startObject("properties") - .startObject("test") - .field("type", "text") - .startObject("fields") - .startObject("tc") - .field("type", "token_count") - .field("analyzer", "standard") - .endObject() - .endObject() - .endObject() - .endObject() - .endObject().endObject()); - - return createIndex("test").mapperService().documentMapperParser().parse("person", new CompressedXContent(content)); + return createDocumentMapper(mapping(b -> { + b.startObject("test"); + { + b.field("type", "text"); + b.startObject("fields"); + { + b.startObject("tc"); + { + b.field("type", "token_count"); + b.field("analyzer", "standard"); + } + b.endObject(); + } + b.endObject(); + } + b.endObject(); + })); } private SourceToParse createDocument(String fieldValue) throws Exception { - BytesReference request = BytesReference.bytes(XContentFactory.jsonBuilder() - .startObject() - .field("test", fieldValue) - .endObject()); - - return new SourceToParse("test", "1", request, XContentType.JSON); + return source(b -> b.field("test", fieldValue)); } private ParseContext.Document parseDocument(DocumentMapper mapper, SourceToParse request) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java index dc5921f23805e..c7401034f002e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java @@ -36,7 +36,6 @@ import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.NumberType; import org.elasticsearch.common.xcontent.XContentParser.Token; @@ -124,9 +123,11 @@ public static class Builder extends ParametrizedFieldMapper.Builder { private final Parameter analyzer; private final Parameter searchAnalyzer; private final Parameter preserveSeparators = Parameter.boolParam("preserve_separators", false, - m -> toType(m).preserveSeparators, Defaults.DEFAULT_PRESERVE_SEPARATORS); + m -> toType(m).preserveSeparators, Defaults.DEFAULT_PRESERVE_SEPARATORS) + .alwaysSerialize(); private final Parameter preservePosInc = Parameter.boolParam("preserve_position_increments", false, - m -> toType(m).preservePosInc, Defaults.DEFAULT_POSITION_INCREMENTS); + m -> toType(m).preservePosInc, Defaults.DEFAULT_POSITION_INCREMENTS) + .alwaysSerialize(); private final Parameter contexts = new Parameter<>("contexts", false, () -> null, (n, c, o) -> ContextMappings.load(o, c.indexVersionCreated()), m -> toType(m).contexts) .setSerializer((b, n, c) -> { @@ -140,7 +141,8 @@ public static class Builder extends ParametrizedFieldMapper.Builder { private final Parameter maxInputLength = Parameter.intParam("max_input_length", true, m -> toType(m).maxInputLength, Defaults.DEFAULT_MAX_INPUT_LENGTH) .addDeprecatedName("max_input_len") - .setValidator(Builder::validateInputLength); + .setValidator(Builder::validateInputLength) + .alwaysSerialize(); private final Parameter> meta = Parameter.metaParam(); private final NamedAnalyzer defaultAnalyzer; @@ -155,7 +157,8 @@ public Builder(String name, NamedAnalyzer defaultAnalyzer, Version indexVersionC super(name); this.defaultAnalyzer = defaultAnalyzer; this.indexVersionCreated = indexVersionCreated; - this.analyzer = Parameter.analyzerParam("analyzer", false, m -> toType(m).analyzer, () -> defaultAnalyzer); + this.analyzer = Parameter.analyzerParam("analyzer", false, m -> toType(m).analyzer, () -> defaultAnalyzer) + .alwaysSerialize(); this.searchAnalyzer = Parameter.analyzerParam("search_analyzer", true, m -> toType(m).searchAnalyzer, analyzer::getValue); } @@ -168,26 +171,7 @@ private static void validateInputLength(int maxInputLength) { @Override protected List> getParameters() { - return List.of(analyzer, searchAnalyzer, preserveSeparators, preservePosInc, contexts, maxInputLength, meta); - } - - @Override - protected void toXContent(XContentBuilder builder, boolean includeDefaults) throws IOException { - builder.field("analyzer", this.analyzer.getValue().name()); - if (Objects.equals(this.analyzer.getValue().name(), this.searchAnalyzer.getValue().name()) == false) { - builder.field("search_analyzer", this.searchAnalyzer.getValue().name()); - } - builder.field(this.preserveSeparators.name, this.preserveSeparators.getValue()); - builder.field(this.preservePosInc.name, this.preservePosInc.getValue()); - builder.field(this.maxInputLength.name, this.maxInputLength.getValue()); - if (this.contexts.getValue() != null) { - builder.startArray(this.contexts.name); - this.contexts.getValue().toXContent(builder, ToXContent.EMPTY_PARAMS); - builder.endArray(); - } - if (this.meta.getValue().isEmpty() == false) { - builder.field(this.meta.name, this.meta.getValue()); - } + return List.of(analyzer, searchAnalyzer, preserveSeparators, preservePosInc, maxInputLength, contexts, meta); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 6bab523d8522e..7e1689f4fc236 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -90,13 +90,8 @@ public static class Builder extends ParametrizedFieldMapper.Builder { private final Parameter indexOptions = Parameter.restrictedStringParam("index_options", false, m -> toType(m).indexOptions, "docs", "freqs"); - private final Parameter hasNorms - = Parameter.boolParam("norms", true, m -> toType(m).fieldType.omitNorms() == false, false) - .setMergeValidator((o, n) -> o == n || (o && n == false)); // norms can be updated from 'true' to 'false' but not vv - private final Parameter similarity = new Parameter<>("similarity", false, () -> null, - (n, c, o) -> TypeParsers.resolveSimilarity(c, n, o), m -> toType(m).similarity) - .setSerializer((b, f, v) -> b.field(f, v == null ? null : v.name()), v -> v == null ? null : v.name()) - .acceptsNull(); + private final Parameter hasNorms = TextParams.norms(false, m -> toType(m).fieldType.omitNorms() == false); + private final Parameter similarity = TextParams.similarity(m -> toType(m).similarity); private final Parameter normalizer = Parameter.stringParam("normalizer", false, m -> toType(m).normalizerName, "default"); @@ -137,19 +132,6 @@ public Builder docValues(boolean hasDocValues) { return this; } - private static IndexOptions toIndexOptions(boolean indexed, String in) { - if (indexed == false) { - return IndexOptions.NONE; - } - switch (in) { - case "docs": - return IndexOptions.DOCS; - case "freqs": - return IndexOptions.DOCS_AND_FREQS; - } - throw new MapperParsingException("Unknown index option [" + in + "]"); - } - @Override protected List> getParameters() { return List.of(indexed, hasDocValues, stored, nullValue, eagerGlobalOrdinals, ignoreAbove, @@ -184,7 +166,7 @@ else if (splitQueriesOnWhitespace.getValue()) { public KeywordFieldMapper build(BuilderContext context) { FieldType fieldtype = new FieldType(Defaults.FIELD_TYPE); fieldtype.setOmitNorms(this.hasNorms.getValue() == false); - fieldtype.setIndexOptions(toIndexOptions(this.indexed.getValue(), this.indexOptions.getValue())); + fieldtype.setIndexOptions(TextParams.toIndexOptions(this.indexed.getValue(), this.indexOptions.getValue())); fieldtype.setStored(this.stored.getValue()); return new KeywordFieldMapper(name, fieldtype, buildFieldType(context, fieldtype), multiFieldsBuilder.build(this, context), copyTo.build(), this); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ParametrizedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ParametrizedFieldMapper.java index a733098243bd8..ff3f9c5912925 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ParametrizedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ParametrizedFieldMapper.java @@ -147,6 +147,7 @@ public static final class Parameter { private Consumer validator = null; private Serializer serializer = XContentBuilder::field; private BooleanSupplier serializerPredicate = () -> true; + private boolean alwaysSerialize = false; private Function conflictSerializer = Objects::toString; private BiPredicate mergeValidator; private T value; @@ -241,6 +242,14 @@ public Parameter setShouldSerialize(BooleanSupplier shouldSerialize) { return this; } + /** + * Ensures that this parameter is always serialized, no matter its value + */ + public Parameter alwaysSerialize() { + this.alwaysSerialize = true; + return this; + } + /** * Sets a custom merge validator. By default, merges are accepted if the * parameter is updateable, or if the previous and new values are equal @@ -275,7 +284,7 @@ private void merge(FieldMapper toMerge, Conflicts conflicts) { } private void toXContent(XContentBuilder builder, boolean includeDefaults) throws IOException { - if ((includeDefaults || isConfigured()) && serializerPredicate.getAsBoolean()) { + if (alwaysSerialize || ((includeDefaults || isConfigured()) && serializerPredicate.getAsBoolean())) { serializer.serialize(builder, name, getValue()); } } @@ -523,7 +532,7 @@ protected String buildFullName(BuilderContext context) { /** * Writes the current builder parameter values as XContent */ - protected void toXContent(XContentBuilder builder, boolean includeDefaults) throws IOException { + protected final void toXContent(XContentBuilder builder, boolean includeDefaults) throws IOException { for (Parameter parameter : getParameters()) { parameter.toXContent(builder, includeDefaults); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextParams.java b/server/src/main/java/org/elasticsearch/index/mapper/TextParams.java new file mode 100644 index 0000000000000..01bce8919e395 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextParams.java @@ -0,0 +1,145 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.document.FieldType; +import org.apache.lucene.index.IndexOptions; +import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.index.mapper.ParametrizedFieldMapper.Parameter; +import org.elasticsearch.index.similarity.SimilarityProvider; + +import java.util.function.Function; +import java.util.function.Supplier; + +/** + * Utility functions for text mapper parameters + */ +public final class TextParams { + + private TextParams() {} + + public static final class Analyzers { + public final Parameter indexAnalyzer; + public final Parameter searchAnalyzer; + public final Parameter searchQuoteAnalyzer; + + public Analyzers(Supplier defaultAnalyzer) { + this.indexAnalyzer = Parameter.analyzerParam("analyzer", false, + m -> m.fieldType().indexAnalyzer(), defaultAnalyzer); + this.searchAnalyzer + = Parameter.analyzerParam("search_analyzer", true, + m -> m.fieldType().getTextSearchInfo().getSearchAnalyzer(), indexAnalyzer::getValue); + this.searchQuoteAnalyzer + = Parameter.analyzerParam("search_quote_analyzer", true, + m -> m.fieldType().getTextSearchInfo().getSearchQuoteAnalyzer(), searchAnalyzer::getValue); + } + + public NamedAnalyzer getIndexAnalyzer() { + return indexAnalyzer.getValue(); + } + + public NamedAnalyzer getSearchAnalyzer() { + return searchAnalyzer.getValue(); + } + + public NamedAnalyzer getSearchQuoteAnalyzer() { + return searchQuoteAnalyzer.getValue(); + } + } + + public static Parameter norms(boolean defaultValue, Function initializer) { + return Parameter.boolParam("norms", true, initializer, defaultValue) + .setMergeValidator((o, n) -> o == n || (o && n == false)); // norms can be updated from 'true' to 'false' but not vv + } + + public static Parameter similarity(Function init) { + return new Parameter<>("similarity", false, () -> null, + (n, c, o) -> TypeParsers.resolveSimilarity(c, n, o), init) + .setSerializer((b, f, v) -> b.field(f, v == null ? null : v.name()), v -> v == null ? null : v.name()) + .acceptsNull(); + } + + public static Parameter indexOptions(Function initializer) { + return Parameter.restrictedStringParam("index_options", false, initializer, + "positions", "docs", "freqs", "offsets"); + } + + public static IndexOptions toIndexOptions(boolean indexed, String indexOptions) { + if (indexed == false) { + return IndexOptions.NONE; + } + switch (indexOptions) { + case "docs": + return IndexOptions.DOCS; + case "freqs": + return IndexOptions.DOCS_AND_FREQS; + case "positions": + return IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; + case "offsets": + return IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS; + } + throw new IllegalArgumentException("Unknown [index_options] value: [" + indexOptions + "]"); + } + + public static Parameter termVectors(Function initializer) { + return Parameter.restrictedStringParam("term_vector", false, initializer, + "no", + "yes", + "with_positions", + "with_offsets", + "with_positions_offsets", + "with_positions_payloads", + "with_positions_offsets_payloads"); + } + + public static void setTermVectorParams(String configuration, FieldType fieldType) { + switch (configuration) { + case "no": + fieldType.setStoreTermVectors(false); + return; + case "yes": + fieldType.setStoreTermVectors(true); + return; + case "with_positions": + fieldType.setStoreTermVectors(true); + fieldType.setStoreTermVectorPositions(true); + return; + case "with_offsets": + case "with_positions_offsets": + fieldType.setStoreTermVectors(true); + fieldType.setStoreTermVectorPositions(true); + fieldType.setStoreTermVectorOffsets(true); + return; + case "with_positions_payloads": + fieldType.setStoreTermVectors(true); + fieldType.setStoreTermVectorPositions(true); + fieldType.setStoreTermVectorPayloads(true); + return; + case "with_positions_offsets_payloads": + fieldType.setStoreTermVectors(true); + fieldType.setStoreTermVectorPositions(true); + fieldType.setStoreTermVectorOffsets(true); + fieldType.setStoreTermVectorPayloads(true); + return; + } + throw new IllegalArgumentException("Unknown [term_vector] setting: [" + configuration + "]"); + } + +}