Skip to content

Commit

Permalink
Fix bugs
Browse files Browse the repository at this point in the history 
Signed-off-by: Rishabh Maurya <[email protected]>
  • Loading branch information
@rishabhmaurya
rishabhmaurya committed Nov 10, 2023
1 parent a1c2fab commit ccab297
Show file tree
Hide file tree
Showing 3 changed files with 112 additions and 16 deletions.
99 changes: 96 additions & 3 deletions server/src/main/java/org/opensearch/index/mapper/MatchOnlyTextFieldMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,17 @@
import org.apache.lucene.search.TermQuery;
import org.opensearch.Version;
import org.opensearch.common.lucene.search.MultiPhrasePrefixQuery;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.index.analysis.IndexAnalyzers;
import org.opensearch.index.analysis.NamedAnalyzer;
import org.opensearch.index.query.QueryShardContext;
import org.opensearch.index.query.SourceFieldMatchQuery;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;

/**
* A specialized type of TextFieldMapper which disables the positions and norms to save on storage and executes phrase queries, which requires
Expand All @@ -38,6 +42,7 @@ public class MatchOnlyTextFieldMapper extends TextFieldMapper {

public static final FieldType FIELD_TYPE = new FieldType();
public static final String CONTENT_TYPE = "match_only_text";
private final String indexOptions = FieldMapper.indexOptionToString(FIELD_TYPE.indexOptions());

@Override
protected String contentType() {
Expand Down Expand Up @@ -69,10 +74,17 @@ protected MatchOnlyTextFieldMapper(
super(simpleName, fieldType, mappedFieldType, prefixFieldMapper, phraseFieldMapper, multiFields, copyTo, builder);
}

@Override
public ParametrizedFieldMapper.Builder getMergeBuilder() {
return new Builder(simpleName(), this.indexCreatedVersion, this.indexAnalyzers).init(this);
}

/**
* Builder class for constructing the MatchOnlyTextFieldMapper.
*/
public static class Builder extends TextFieldMapper.Builder {
final Parameter<String> indexOptions = TextParams.indexOptions(m -> ((MatchOnlyTextFieldMapper) m).indexOptions);
final Parameter<Boolean> norms = TextParams.norms(true, m -> ((MatchOnlyTextFieldMapper) m).fieldType.omitNorms() == false);

public Builder(String name, IndexAnalyzers indexAnalyzers) {
super(name, indexAnalyzers);
Expand All @@ -84,8 +96,9 @@ public Builder(String name, Version indexCreatedVersion, IndexAnalyzers indexAna

@Override
public MatchOnlyTextFieldMapper build(BuilderContext context) {
// TODO - disable norms and index-options and validate
FieldType fieldType = FIELD_TYPE;
MatchOnlyTextFieldType tft = new MatchOnlyTextFieldType(buildFieldType(fieldType, context));
MatchOnlyTextFieldType tft = buildFieldType(fieldType, context);
return new MatchOnlyTextFieldMapper(
name,
fieldType,
Expand All @@ -97,6 +110,60 @@ public MatchOnlyTextFieldMapper build(BuilderContext context) {
this
);
}

@Override
protected MatchOnlyTextFieldType buildFieldType(FieldType fieldType, BuilderContext context) {
NamedAnalyzer indexAnalyzer = analyzers.getIndexAnalyzer();
NamedAnalyzer searchAnalyzer = analyzers.getSearchAnalyzer();
NamedAnalyzer searchQuoteAnalyzer = analyzers.getSearchQuoteAnalyzer();

if (fieldType.indexOptions().compareTo(IndexOptions.DOCS) != 0) {
throw new IllegalArgumentException("Cannot set position_increment_gap on field [" + name + "] without positions enabled");
}
if (positionIncrementGap.get() != POSITION_INCREMENT_GAP_USE_ANALYZER) {
indexAnalyzer = new NamedAnalyzer(indexAnalyzer, positionIncrementGap.get());
searchAnalyzer = new NamedAnalyzer(searchAnalyzer, positionIncrementGap.get());
searchQuoteAnalyzer = new NamedAnalyzer(searchQuoteAnalyzer, positionIncrementGap.get());
}
TextSearchInfo tsi = new TextSearchInfo(fieldType, similarity.getValue(), searchAnalyzer, searchQuoteAnalyzer);
MatchOnlyTextFieldType ft = new MatchOnlyTextFieldType(
buildFullName(context),
index.getValue(),
fieldType.stored(),
tsi,
meta.getValue()
);
ft.setIndexAnalyzer(indexAnalyzer);
ft.setEagerGlobalOrdinals(eagerGlobalOrdinals.getValue());
ft.setBoost(boost.getValue());
if (fieldData.getValue()) {
ft.setFielddata(true, freqFilter.getValue());
}
return ft;
}

@Override
protected List<Parameter<?>> getParameters() {
return Arrays.asList(
index,
store,
indexOptions,
norms,
termVectors,
analyzers.indexAnalyzer,
analyzers.searchAnalyzer,
analyzers.searchQuoteAnalyzer,
similarity,
positionIncrementGap,
fieldData,
freqFilter,
eagerGlobalOrdinals,
indexPhrases,
indexPrefixes,
boost,
meta
);
}
}

/**
Expand All @@ -111,8 +178,8 @@ public String typeName() {
return CONTENT_TYPE;
}

public MatchOnlyTextFieldType(TextFieldMapper.TextFieldType tft) {
super(tft.name(), tft.isSearchable(), tft.isStored(), tft.getTextSearchInfo(), tft.meta());
public MatchOnlyTextFieldType(String name, boolean indexed, boolean stored, TextSearchInfo tsi, Map<String, String> meta) {
super(name, indexed, stored, tsi, meta);
}

@Override
Expand Down Expand Up @@ -198,4 +265,30 @@ private List<List<Term>> getTermsFromTokenStream(TokenStream stream) throws IOEx
return termArray;
}
}

@Override
protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
// this is a pain, but we have to do this to maintain BWC
builder.field("type", contentType());
Builder mapperBuilder = (MatchOnlyTextFieldMapper.Builder) getMergeBuilder();
mapperBuilder.boost.toXContent(builder, includeDefaults);
mapperBuilder.index.toXContent(builder, includeDefaults);
mapperBuilder.store.toXContent(builder, includeDefaults);
this.multiFields.toXContent(builder, params);
this.copyTo.toXContent(builder, params);
mapperBuilder.meta.toXContent(builder, includeDefaults);
mapperBuilder.indexOptions.toXContent(builder, includeDefaults);
mapperBuilder.termVectors.toXContent(builder, includeDefaults);
mapperBuilder.norms.toXContent(builder, includeDefaults);
mapperBuilder.analyzers.indexAnalyzer.toXContent(builder, includeDefaults);
mapperBuilder.analyzers.searchAnalyzer.toXContent(builder, includeDefaults);
mapperBuilder.analyzers.searchQuoteAnalyzer.toXContent(builder, includeDefaults);
mapperBuilder.similarity.toXContent(builder, includeDefaults);
mapperBuilder.eagerGlobalOrdinals.toXContent(builder, includeDefaults);
mapperBuilder.positionIncrementGap.toXContent(builder, includeDefaults);
mapperBuilder.fieldData.toXContent(builder, includeDefaults);
mapperBuilder.freqFilter.toXContent(builder, includeDefaults);
mapperBuilder.indexPrefixes.toXContent(builder, includeDefaults);
mapperBuilder.indexPhrases.toXContent(builder, includeDefaults);
}
}
19 changes: 9 additions & 10 deletions server/src/main/java/org/opensearch/index/mapper/TextFieldMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@
public class TextFieldMapper extends ParametrizedFieldMapper {

public static final String CONTENT_TYPE = "text";
private static final int POSITION_INCREMENT_GAP_USE_ANALYZER = -1;
protected static final int POSITION_INCREMENT_GAP_USE_ANALYZER = -1;
private static final String FAST_PHRASE_SUFFIX = "._index_phrase";

/**
Expand Down Expand Up @@ -214,7 +214,7 @@ private static PrefixConfig parsePrefixConfig(String propName, ParserContext par
*
* @opensearch.internal
*/
private static final class FielddataFrequencyFilter implements ToXContent {
protected static final class FielddataFrequencyFilter implements ToXContent {
final double minFreq;
final double maxFreq;
final int minSegmentSize;
Expand Down Expand Up @@ -280,15 +280,14 @@ public static class Builder extends ParametrizedFieldMapper.Builder {

private final Version indexCreatedVersion;

private final Parameter<Boolean> index = Parameter.indexParam(m -> toType(m).mappedFieldType.isSearchable(), true);
private final Parameter<Boolean> store = Parameter.storeParam(m -> toType(m).fieldType.stored(), false);
protected final Parameter<Boolean> index = Parameter.indexParam(m -> toType(m).mappedFieldType.isSearchable(), true);
protected final Parameter<Boolean> store = Parameter.storeParam(m -> toType(m).fieldType.stored(), false);

final Parameter<SimilarityProvider> similarity = TextParams.similarity(m -> toType(m).similarity);

final Parameter<String> indexOptions = TextParams.indexOptions(m -> toType(m).indexOptions);
final Parameter<Boolean> norms = TextParams.norms(true, m -> toType(m).fieldType.omitNorms() == false);
final Parameter<String> termVectors = TextParams.termVectors(m -> toType(m).termVectors);

final Parameter<Integer> positionIncrementGap = Parameter.intParam(
"position_increment_gap",
false,
Expand Down Expand Up @@ -332,8 +331,8 @@ public static class Builder extends ParametrizedFieldMapper.Builder {
.orElse(null)
).acceptsNull();

private final Parameter<Float> boost = Parameter.boostParam();
private final Parameter<Map<String, String>> meta = Parameter.metaParam();
protected final Parameter<Float> boost = Parameter.boostParam();
protected final Parameter<Map<String, String>> meta = Parameter.metaParam();

final TextParams.Analyzers analyzers;

Expand Down Expand Up @@ -968,15 +967,15 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S

}

private final FieldType fieldType;
protected final FieldType fieldType;
private final PrefixFieldMapper prefixFieldMapper;
private final PhraseFieldMapper phraseFieldMapper;
private final SimilarityProvider similarity;
private final String indexOptions;
private final String termVectors;
private final int positionIncrementGap;
private final Version indexCreatedVersion;
private final IndexAnalyzers indexAnalyzers;
protected final Version indexCreatedVersion;
protected final IndexAnalyzers indexAnalyzers;
private final FielddataFrequencyFilter freqFilter;

protected TextFieldMapper(
Expand Down
10 changes: 7 additions & 3 deletions server/src/main/java/org/opensearch/index/query/SourceFieldMatchQuery.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,12 @@ public void visit(QueryVisitor visitor) {
}

@Override
public Query rewrite(IndexSearcher searcher) throws IOException {
return delegateQuery.rewrite(searcher);
public Query rewrite(IndexSearcher indexSearcher) throws IOException {
Query rewritten = indexSearcher.rewrite(delegateQuery);
if (rewritten == delegateQuery) {
return this;
}
return new SourceFieldMatchQuery(rewritten, filter, fieldType, valueFetcher, lookup);
}

@Override
Expand All @@ -96,7 +100,7 @@ public boolean matches() {
for (Object value : values) {
memoryIndex.addField(fieldType.name(), (String) value, fieldType.indexAnalyzer());
}
float score = memoryIndex.search(delegateQuery);
float score = memoryIndex.search(filter);
return score > 0.0f;
}

Expand Down

0 comments on commit ccab297

Please sign in to comment.