Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Rescore collapsed documents #28521

Merged
merged 14 commits into from
Mar 4, 2018
Merged

Rescore collapsed documents #28521

Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
90cb54e
rescore in collapse top doc collector
fred84 Jan 30, 2018
a770c25
Merge remote-tracking branch 'upstream/master' into 27243_collapse_wi…
fred84 Feb 1, 2018
e235174
Merge remote-tracking branch 'upstream/master' into 27243_collapse_wi…
fred84 Feb 2, 2018
e737519
test for rescore + collapse
fred84 Feb 2, 2018
0b91413
collapse + rescore test
fred84 Feb 5, 2018
40fca97
Merge remote-tracking branch 'upstream/master' into 27243_collapse_wi…
fred84 Feb 5, 2018
5663f10
Merge remote-tracking branch 'upstream/master' into 27243_collapse_wi…
fred84 Feb 5, 2018
a5d8834
Merge branch '27242_collapse_with_rescore' of https://github.com/fred…
fred84 Feb 5, 2018
e060df4
clean diff
fred84 Feb 5, 2018
4f5729a
fix checkstyle warnings
fred84 Feb 5, 2018
cc8f8ef
fix test
fred84 Feb 5, 2018
12e4d06
Merge branch 'master' into 27243_collapse_with_rescore
fred84 Feb 26, 2018
c5151b5
Merge branch '27243_collapse_with_rescore' of https://github.com/fred…
fred84 Feb 26, 2018
8320622
remove rest test + static score in integration test
fred84 Feb 27, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions server/src/main/java/org/elasticsearch/search/collapse/CollapseBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -226,9 +226,6 @@ public CollapseContext build(SearchContext context) {
if (context.searchAfter() != null) {
throw new SearchContextException(context, "cannot use `collapse` in conjunction with `search_after`");
}
if (context.rescore() != null && context.rescore().isEmpty() == false) {
throw new SearchContextException(context, "cannot use `collapse` in conjunction with `rescore`");
}

MappedFieldType fieldType = context.getQueryShardContext().fieldMapper(field);
if (fieldType == null) {
Expand Down
19 changes: 13 additions & 6 deletions server/src/main/java/org/elasticsearch/search/query/TopDocsCollectorContext.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,7 @@ void postProcess(QuerySearchResult result) {
static class CollapsingTopDocsCollectorContext extends TopDocsCollectorContext {
private final DocValueFormat[] sortFmt;
private final CollapsingTopDocsCollector<?> topDocsCollector;
private final boolean rescore;

/**
* Ctr
Expand All @@ -139,13 +140,14 @@ static class CollapsingTopDocsCollectorContext extends TopDocsCollectorContext {
private CollapsingTopDocsCollectorContext(CollapseContext collapseContext,
@Nullable SortAndFormats sortAndFormats,
int numHits,
boolean trackMaxScore) {
boolean trackMaxScore, boolean rescore) {
super(REASON_SEARCH_TOP_HITS, numHits);
assert numHits > 0;
assert collapseContext != null;
Sort sort = sortAndFormats == null ? Sort.RELEVANCE : sortAndFormats.sort;
this.sortFmt = sortAndFormats == null ? new DocValueFormat[] { DocValueFormat.RAW } : sortAndFormats.formats;
this.topDocsCollector = collapseContext.createTopDocs(sort, numHits, trackMaxScore);
this.rescore = rescore;
}

@Override
Expand All @@ -158,6 +160,11 @@ Collector create(Collector in) throws IOException {
void postProcess(QuerySearchResult result) throws IOException {
result.topDocs(topDocsCollector.getTopDocs(), sortFmt);
}

@Override
boolean shouldRescore() {
return rescore;
}
}

abstract static class SimpleTopDocsCollectorContext extends TopDocsCollectorContext {
Expand Down Expand Up @@ -332,11 +339,6 @@ static TopDocsCollectorContext createTopDocsCollectorContext(SearchContext searc
return new ScrollingTopDocsCollectorContext(reader, query, searchContext.scrollContext(),
searchContext.sort(), numDocs, searchContext.trackScores(), searchContext.numberOfShards(),
searchContext.trackTotalHits(), hasFilterCollector);
} else if (searchContext.collapse() != null) {
boolean trackScores = searchContext.sort() == null ? true : searchContext.trackScores();
int numDocs = Math.min(searchContext.from() + searchContext.size(), totalNumDocs);
return new CollapsingTopDocsCollectorContext(searchContext.collapse(),
searchContext.sort(), numDocs, trackScores);
} else {
int numDocs = Math.min(searchContext.from() + searchContext.size(), totalNumDocs);
final boolean rescore = searchContext.rescore().isEmpty() == false;
Expand All @@ -346,6 +348,11 @@ static TopDocsCollectorContext createTopDocsCollectorContext(SearchContext searc
numDocs = Math.max(numDocs, rescoreContext.getWindowSize());
}
}
if (searchContext.collapse() != null) {
boolean trackScores = searchContext.sort() == null ? true : searchContext.trackScores();
return new CollapsingTopDocsCollectorContext(searchContext.collapse(),
searchContext.sort(), numDocs, trackScores, rescore);
}
return new SimpleTopDocsCollectorContext(reader, query, searchContext.sort(), searchContext.searchAfter(), numDocs,
searchContext.trackScores(), searchContext.trackTotalHits(), hasFilterCollector) {
@Override
Expand Down
57 changes: 57 additions & 0 deletions server/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,19 +30,24 @@
import org.elasticsearch.common.settings.Settings.Builder;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.MatchQueryBuilder;
import org.elasticsearch.index.query.Operator;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.collapse.CollapseBuilder;
import org.elasticsearch.search.rescore.QueryRescoreMode;
import org.elasticsearch.search.rescore.QueryRescorerBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.test.ESIntegTestCase;

import java.io.IOException;
import java.util.Arrays;
import java.util.Comparator;
import java.util.Map;
import java.util.stream.Collectors;

import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS;
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS;
Expand All @@ -67,6 +72,7 @@
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThirdHit;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasScore;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
Expand Down Expand Up @@ -748,4 +754,55 @@ public void testRescorePhaseWithInvalidSort() throws Exception {
assertThat(hit.getScore(), equalTo(101f));
}
}

public void testRescoreAfterCollapse() throws Exception {
assertAcked(prepareCreate("test")
.addMapping(
"type1",
jsonBuilder()
.startObject()
.startObject("properties")
.startObject("group")
.field("type", "keyword")
.endObject()
.endObject()
.endObject())
);

indexDocument(1, "value", "a");
indexDocument(2, "one one value", "a");
indexDocument(3, "one one two value", "b");
// should be highest on rescore, but filtered out during collapse
indexDocument(4, "one two two value", "b");

refresh("test");

SearchResponse searchResponse = client().prepareSearch("test")
.setTypes("type1")
.setQuery(new MatchQueryBuilder("name", "one"))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The score of this query depends on the number of shards, the default similarity, ... To make sure that we have consistent scoring you can use a function_score query like the following:

 QueryBuilder query = functionScoreQuery(
            termQuery("name", "one"),
            ScoreFunctionBuilders.fieldValueFactorFunction("my_static_doc_score") 
        ).boostMode(CombineFunction.REPLACE);

... and add the my_static_doc_score at indexing time.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fixed

.addRescorer(new QueryRescorerBuilder(new MatchQueryBuilder("name", "two")))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can use the same for the rescore with another field for instance

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fixed

.setCollapse(new CollapseBuilder("group"))
.execute()
.actionGet();

assertThat(searchResponse.getHits().totalHits, equalTo(3L));
assertThat(searchResponse.getHits().getHits().length, equalTo(2));

Map<String, Float> collapsedHits = Arrays
.stream(searchResponse.getHits().getHits())
.collect(Collectors.toMap(SearchHit::getId, SearchHit::getScore));

assertThat(collapsedHits.keySet(), containsInAnyOrder("2", "3"));
assertThat(collapsedHits.get("3"), greaterThan(collapsedHits.get("2")));
}

private void indexDocument(int id, String name, String group) throws IOException {
XContentBuilder docBuilder =jsonBuilder()
.startObject()
.field("name", name)
.field("group", group)
.endObject();

client().prepareIndex("test", "type1", Integer.toString(id)).setSource(docBuilder).execute().actionGet();
}
}