diff --git a/server/src/main/java/org/opensearch/index/mapper/MatchOnlyTextFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/MatchOnlyTextFieldMapper.java index ad6eab020ef82..199ecad8028b9 100644 --- a/server/src/main/java/org/opensearch/index/mapper/MatchOnlyTextFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/MatchOnlyTextFieldMapper.java @@ -22,13 +22,17 @@ import org.apache.lucene.search.TermQuery; import org.opensearch.Version; import org.opensearch.common.lucene.search.MultiPhrasePrefixQuery; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.index.analysis.IndexAnalyzers; +import org.opensearch.index.analysis.NamedAnalyzer; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.query.SourceFieldMatchQuery; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import java.util.Map; /** * A specialized type of TextFieldMapper which disables the positions and norms to save on storage and executes phrase queries, which requires @@ -38,6 +42,7 @@ public class MatchOnlyTextFieldMapper extends TextFieldMapper { public static final FieldType FIELD_TYPE = new FieldType(); public static final String CONTENT_TYPE = "match_only_text"; + private final String indexOptions = FieldMapper.indexOptionToString(FIELD_TYPE.indexOptions()); @Override protected String contentType() { @@ -69,10 +74,17 @@ protected MatchOnlyTextFieldMapper( super(simpleName, fieldType, mappedFieldType, prefixFieldMapper, phraseFieldMapper, multiFields, copyTo, builder); } + @Override + public ParametrizedFieldMapper.Builder getMergeBuilder() { + return new Builder(simpleName(), this.indexCreatedVersion, this.indexAnalyzers).init(this); + } + /** * Builder class for constructing the MatchOnlyTextFieldMapper. */ public static class Builder extends TextFieldMapper.Builder { + final Parameter indexOptions = TextParams.indexOptions(m -> ((MatchOnlyTextFieldMapper) m).indexOptions); + final Parameter norms = TextParams.norms(true, m -> ((MatchOnlyTextFieldMapper) m).fieldType.omitNorms() == false); public Builder(String name, IndexAnalyzers indexAnalyzers) { super(name, indexAnalyzers); @@ -84,8 +96,9 @@ public Builder(String name, Version indexCreatedVersion, IndexAnalyzers indexAna @Override public MatchOnlyTextFieldMapper build(BuilderContext context) { + // TODO - disable norms and index-options and validate FieldType fieldType = FIELD_TYPE; - MatchOnlyTextFieldType tft = new MatchOnlyTextFieldType(buildFieldType(fieldType, context)); + MatchOnlyTextFieldType tft = buildFieldType(fieldType, context); return new MatchOnlyTextFieldMapper( name, fieldType, @@ -97,6 +110,60 @@ public MatchOnlyTextFieldMapper build(BuilderContext context) { this ); } + + @Override + protected MatchOnlyTextFieldType buildFieldType(FieldType fieldType, BuilderContext context) { + NamedAnalyzer indexAnalyzer = analyzers.getIndexAnalyzer(); + NamedAnalyzer searchAnalyzer = analyzers.getSearchAnalyzer(); + NamedAnalyzer searchQuoteAnalyzer = analyzers.getSearchQuoteAnalyzer(); + + if (fieldType.indexOptions().compareTo(IndexOptions.DOCS) != 0) { + throw new IllegalArgumentException("Cannot set position_increment_gap on field [" + name + "] without positions enabled"); + } + if (positionIncrementGap.get() != POSITION_INCREMENT_GAP_USE_ANALYZER) { + indexAnalyzer = new NamedAnalyzer(indexAnalyzer, positionIncrementGap.get()); + searchAnalyzer = new NamedAnalyzer(searchAnalyzer, positionIncrementGap.get()); + searchQuoteAnalyzer = new NamedAnalyzer(searchQuoteAnalyzer, positionIncrementGap.get()); + } + TextSearchInfo tsi = new TextSearchInfo(fieldType, similarity.getValue(), searchAnalyzer, searchQuoteAnalyzer); + MatchOnlyTextFieldType ft = new MatchOnlyTextFieldType( + buildFullName(context), + index.getValue(), + fieldType.stored(), + tsi, + meta.getValue() + ); + ft.setIndexAnalyzer(indexAnalyzer); + ft.setEagerGlobalOrdinals(eagerGlobalOrdinals.getValue()); + ft.setBoost(boost.getValue()); + if (fieldData.getValue()) { + ft.setFielddata(true, freqFilter.getValue()); + } + return ft; + } + + @Override + protected List> getParameters() { + return Arrays.asList( + index, + store, + indexOptions, + norms, + termVectors, + analyzers.indexAnalyzer, + analyzers.searchAnalyzer, + analyzers.searchQuoteAnalyzer, + similarity, + positionIncrementGap, + fieldData, + freqFilter, + eagerGlobalOrdinals, + indexPhrases, + indexPrefixes, + boost, + meta + ); + } } /** @@ -111,8 +178,8 @@ public String typeName() { return CONTENT_TYPE; } - public MatchOnlyTextFieldType(TextFieldMapper.TextFieldType tft) { - super(tft.name(), tft.isSearchable(), tft.isStored(), tft.getTextSearchInfo(), tft.meta()); + public MatchOnlyTextFieldType(String name, boolean indexed, boolean stored, TextSearchInfo tsi, Map meta) { + super(name, indexed, stored, tsi, meta); } @Override @@ -198,4 +265,30 @@ private List> getTermsFromTokenStream(TokenStream stream) throws IOEx return termArray; } } + + @Override + protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { + // this is a pain, but we have to do this to maintain BWC + builder.field("type", contentType()); + Builder mapperBuilder = (MatchOnlyTextFieldMapper.Builder) getMergeBuilder(); + mapperBuilder.boost.toXContent(builder, includeDefaults); + mapperBuilder.index.toXContent(builder, includeDefaults); + mapperBuilder.store.toXContent(builder, includeDefaults); + this.multiFields.toXContent(builder, params); + this.copyTo.toXContent(builder, params); + mapperBuilder.meta.toXContent(builder, includeDefaults); + mapperBuilder.indexOptions.toXContent(builder, includeDefaults); + mapperBuilder.termVectors.toXContent(builder, includeDefaults); + mapperBuilder.norms.toXContent(builder, includeDefaults); + mapperBuilder.analyzers.indexAnalyzer.toXContent(builder, includeDefaults); + mapperBuilder.analyzers.searchAnalyzer.toXContent(builder, includeDefaults); + mapperBuilder.analyzers.searchQuoteAnalyzer.toXContent(builder, includeDefaults); + mapperBuilder.similarity.toXContent(builder, includeDefaults); + mapperBuilder.eagerGlobalOrdinals.toXContent(builder, includeDefaults); + mapperBuilder.positionIncrementGap.toXContent(builder, includeDefaults); + mapperBuilder.fieldData.toXContent(builder, includeDefaults); + mapperBuilder.freqFilter.toXContent(builder, includeDefaults); + mapperBuilder.indexPrefixes.toXContent(builder, includeDefaults); + mapperBuilder.indexPhrases.toXContent(builder, includeDefaults); + } } diff --git a/server/src/main/java/org/opensearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/TextFieldMapper.java index 5780e42105fe5..ffef9044bd715 100644 --- a/server/src/main/java/org/opensearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/TextFieldMapper.java @@ -110,7 +110,7 @@ public class TextFieldMapper extends ParametrizedFieldMapper { public static final String CONTENT_TYPE = "text"; - private static final int POSITION_INCREMENT_GAP_USE_ANALYZER = -1; + protected static final int POSITION_INCREMENT_GAP_USE_ANALYZER = -1; private static final String FAST_PHRASE_SUFFIX = "._index_phrase"; /** @@ -214,7 +214,7 @@ private static PrefixConfig parsePrefixConfig(String propName, ParserContext par * * @opensearch.internal */ - private static final class FielddataFrequencyFilter implements ToXContent { + protected static final class FielddataFrequencyFilter implements ToXContent { final double minFreq; final double maxFreq; final int minSegmentSize; @@ -280,15 +280,14 @@ public static class Builder extends ParametrizedFieldMapper.Builder { private final Version indexCreatedVersion; - private final Parameter index = Parameter.indexParam(m -> toType(m).mappedFieldType.isSearchable(), true); - private final Parameter store = Parameter.storeParam(m -> toType(m).fieldType.stored(), false); + protected final Parameter index = Parameter.indexParam(m -> toType(m).mappedFieldType.isSearchable(), true); + protected final Parameter store = Parameter.storeParam(m -> toType(m).fieldType.stored(), false); final Parameter similarity = TextParams.similarity(m -> toType(m).similarity); final Parameter indexOptions = TextParams.indexOptions(m -> toType(m).indexOptions); final Parameter norms = TextParams.norms(true, m -> toType(m).fieldType.omitNorms() == false); final Parameter termVectors = TextParams.termVectors(m -> toType(m).termVectors); - final Parameter positionIncrementGap = Parameter.intParam( "position_increment_gap", false, @@ -332,8 +331,8 @@ public static class Builder extends ParametrizedFieldMapper.Builder { .orElse(null) ).acceptsNull(); - private final Parameter boost = Parameter.boostParam(); - private final Parameter> meta = Parameter.metaParam(); + protected final Parameter boost = Parameter.boostParam(); + protected final Parameter> meta = Parameter.metaParam(); final TextParams.Analyzers analyzers; @@ -968,15 +967,15 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S } - private final FieldType fieldType; + protected final FieldType fieldType; private final PrefixFieldMapper prefixFieldMapper; private final PhraseFieldMapper phraseFieldMapper; private final SimilarityProvider similarity; private final String indexOptions; private final String termVectors; private final int positionIncrementGap; - private final Version indexCreatedVersion; - private final IndexAnalyzers indexAnalyzers; + protected final Version indexCreatedVersion; + protected final IndexAnalyzers indexAnalyzers; private final FielddataFrequencyFilter freqFilter; protected TextFieldMapper( diff --git a/server/src/main/java/org/opensearch/index/query/SourceFieldMatchQuery.java b/server/src/main/java/org/opensearch/index/query/SourceFieldMatchQuery.java index 297cf5fd0e7ee..841cbcf23f922 100644 --- a/server/src/main/java/org/opensearch/index/query/SourceFieldMatchQuery.java +++ b/server/src/main/java/org/opensearch/index/query/SourceFieldMatchQuery.java @@ -69,8 +69,12 @@ public void visit(QueryVisitor visitor) { } @Override - public Query rewrite(IndexSearcher searcher) throws IOException { - return delegateQuery.rewrite(searcher); + public Query rewrite(IndexSearcher indexSearcher) throws IOException { + Query rewritten = indexSearcher.rewrite(delegateQuery); + if (rewritten == delegateQuery) { + return this; + } + return new SourceFieldMatchQuery(rewritten, filter, fieldType, valueFetcher, lookup); } @Override @@ -96,7 +100,7 @@ public boolean matches() { for (Object value : values) { memoryIndex.addField(fieldType.name(), (String) value, fieldType.indexAnalyzer()); } - float score = memoryIndex.search(delegateQuery); + float score = memoryIndex.search(filter); return score > 0.0f; }