Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Pruning of estimating the point value count in BooleanScorerSupplier #13988

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Pruning of estimating the point value count in BooleanScorerSupplier #13988

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 25 additions & 5 deletions lucene/core/src/java/org/apache/lucene/document/LatLonPointDistanceQuery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
import static org.apache.lucene.geo.GeoEncodingUtils.decodeLongitude;
import static org.apache.lucene.geo.GeoEncodingUtils.encodeLatitude;
import static org.apache.lucene.geo.GeoEncodingUtils.encodeLongitude;
import static org.apache.lucene.search.TotalHits.Relation.EQUAL_TO;
import static org.apache.lucene.search.TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO;

import java.io.IOException;
import org.apache.lucene.geo.GeoEncodingUtils;
Expand All @@ -40,6 +42,7 @@
import org.apache.lucene.search.ScoreMode;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.ScorerSupplier;
import org.apache.lucene.search.TotalHits;
import org.apache.lucene.search.Weight;
import org.apache.lucene.util.BitSetIterator;
import org.apache.lucene.util.DocIdSetBuilder;
Expand Down Expand Up @@ -139,7 +142,7 @@ public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOExcepti

return new ScorerSupplier() {

long cost = -1;
TotalHits estimatedCount;

@Override
public Scorer get(long leadCost) throws IOException {
Expand All @@ -162,11 +165,28 @@ && cost() > reader.maxDoc() / 2) {

@Override
public long cost() {
if (cost == -1) {
cost = values.estimateDocCount(visitor);
if (estimatedCount == null || estimatedCount.relation() == GREATER_THAN_OR_EQUAL_TO) {
estimatedCount =
new TotalHits(values.estimateDocCount(visitor, Long.MAX_VALUE), EQUAL_TO);
assert estimatedCount.value() >= 0;
}
assert cost >= 0;
return cost;
return estimatedCount.value();
}

@Override
public TotalHits isEstimatedPointCountGreaterThanOrEqualTo(long upperBound) {
if (estimatedCount == null
|| (estimatedCount.value() < upperBound
&& estimatedCount.relation() == GREATER_THAN_OR_EQUAL_TO)) {
long cost = values.estimateDocCount(visitor, Long.MAX_VALUE);
if (cost < upperBound) {
estimatedCount = new TotalHits(cost, EQUAL_TO);
} else if (estimatedCount == null || cost > estimatedCount.value()) {
estimatedCount = new TotalHits(cost, GREATER_THAN_OR_EQUAL_TO);
}
assert estimatedCount.value() >= 0;
}
return estimatedCount;
}
};
}
Expand Down
31 changes: 26 additions & 5 deletions lucene/core/src/java/org/apache/lucene/document/RangeFieldQuery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@
*/
package org.apache.lucene.document;

import static org.apache.lucene.search.TotalHits.Relation.EQUAL_TO;
import static org.apache.lucene.search.TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO;

import java.io.IOException;
import java.util.Arrays;
import java.util.Objects;
Expand All @@ -34,6 +37,7 @@
import org.apache.lucene.search.ScoreMode;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.ScorerSupplier;
import org.apache.lucene.search.TotalHits;
import org.apache.lucene.search.Weight;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.ArrayUtil.ByteArrayComparator;
Expand Down Expand Up @@ -477,7 +481,7 @@ public long cost() {

final DocIdSetBuilder result = new DocIdSetBuilder(reader.maxDoc(), values, field);
final IntersectVisitor visitor = getIntersectVisitor(result);
long cost = -1;
TotalHits estimatedCount = null;

@Override
public Scorer get(long leadCost) throws IOException {
Expand All @@ -488,12 +492,29 @@ public Scorer get(long leadCost) throws IOException {

@Override
public long cost() {
if (cost == -1) {
if (estimatedCount == null || estimatedCount.relation() == GREATER_THAN_OR_EQUAL_TO) {
estimatedCount =
new TotalHits(values.estimateDocCount(visitor, Long.MAX_VALUE), EQUAL_TO);
assert estimatedCount.value() >= 0;
}
return estimatedCount.value();
}

@Override
public TotalHits isEstimatedPointCountGreaterThanOrEqualTo(long upperBound) {
if (estimatedCount == null
|| (estimatedCount.value() < upperBound
&& estimatedCount.relation() == GREATER_THAN_OR_EQUAL_TO)) {
// Computing the cost may be expensive, so only do it if necessary
cost = values.estimateDocCount(visitor);
assert cost >= 0;
long cost = values.estimateDocCount(visitor, upperBound);
if (cost < upperBound) {
estimatedCount = new TotalHits(cost, EQUAL_TO);
} else if (estimatedCount == null || cost > estimatedCount.value()) {
estimatedCount = new TotalHits(cost, GREATER_THAN_OR_EQUAL_TO);
}
assert estimatedCount.value() >= 0;
}
return cost;
return estimatedCount;
}
};
}
Expand Down
31 changes: 26 additions & 5 deletions lucene/core/src/java/org/apache/lucene/document/XYPointInGeometryQuery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@
*/
package org.apache.lucene.document;

import static org.apache.lucene.search.TotalHits.Relation.EQUAL_TO;
import static org.apache.lucene.search.TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO;

import java.io.IOException;
import java.util.Arrays;
import org.apache.lucene.geo.Component2D;
Expand All @@ -36,6 +39,7 @@
import org.apache.lucene.search.ScoreMode;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.ScorerSupplier;
import org.apache.lucene.search.TotalHits;
import org.apache.lucene.search.Weight;
import org.apache.lucene.util.DocIdSetBuilder;

Expand Down Expand Up @@ -144,7 +148,7 @@ public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOExcepti

return new ScorerSupplier() {

long cost = -1;
TotalHits estimatedCount;
DocIdSetBuilder result = new DocIdSetBuilder(reader.maxDoc(), values, field);
final IntersectVisitor visitor = getIntersectVisitor(result, tree);

Expand All @@ -156,12 +160,29 @@ public Scorer get(long leadCost) throws IOException {

@Override
public long cost() {
if (cost == -1) {
if (estimatedCount == null || estimatedCount.relation() == GREATER_THAN_OR_EQUAL_TO) {
// Computing the cost may be expensive, so only do it if necessary
cost = values.estimateDocCount(visitor);
assert cost >= 0;
estimatedCount =
new TotalHits(values.estimateDocCount(visitor, Long.MAX_VALUE), EQUAL_TO);
assert estimatedCount.value() >= 0;
}
return estimatedCount.value();
}

@Override
public TotalHits isEstimatedPointCountGreaterThanOrEqualTo(long upperBound) {
if (estimatedCount == null
|| (estimatedCount.value() < upperBound
&& estimatedCount.relation() == GREATER_THAN_OR_EQUAL_TO)) {
long cost = values.estimateDocCount(visitor, upperBound);
if (cost < upperBound) {
estimatedCount = new TotalHits(cost, EQUAL_TO);
} else if (estimatedCount == null || cost > estimatedCount.value()) {
estimatedCount = new TotalHits(cost, GREATER_THAN_OR_EQUAL_TO);
}
assert estimatedCount.value() >= 0;
}
return cost;
return estimatedCount;
}
};
}
Expand Down
20 changes: 18 additions & 2 deletions lucene/core/src/java/org/apache/lucene/index/PointValues.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -385,9 +385,17 @@ private void intersect(IntersectVisitor visitor, PointTree pointTree) throws IOE
* IntersectVisitor}. This should run many times faster than {@link #intersect(IntersectVisitor)}.
*/
public final long estimatePointCount(IntersectVisitor visitor) {
return estimatePointCount(visitor, Long.MAX_VALUE);
}

/**
* Estimate the number of points within the given {@link IntersectVisitor} and a maximum of
* {upperBound}
*/
public final long estimatePointCount(IntersectVisitor visitor, long upperBound) {
try {
final PointTree pointTree = getPointTree();
final long count = estimatePointCount(visitor, pointTree, Long.MAX_VALUE);
final long count = estimatePointCount(visitor, pointTree, upperBound);
assert pointTree.moveToParent() == false;
return count;
} catch (IOException ioe) {
Expand Down Expand Up @@ -449,7 +457,15 @@ private static long estimatePointCount(
* @see DocIdSetIterator#cost
*/
public final long estimateDocCount(IntersectVisitor visitor) {
long estimatedPointCount = estimatePointCount(visitor);
return estimateDocCount(visitor, Long.MAX_VALUE);
}

/**
* Estimate the number of documents that would be matched by {@link #intersect} with the given
* {upperBound}
*/
public final long estimateDocCount(IntersectVisitor visitor, long upperBound) {
long estimatedPointCount = estimatePointCount(visitor, upperBound);
int docCount = getDocCount();
double size = size();
if (estimatedPointCount >= size) {
Expand Down
92 changes: 68 additions & 24 deletions lucene/core/src/java/org/apache/lucene/search/BooleanScorerSupplier.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@
*/
package org.apache.lucene.search;

import static org.apache.lucene.search.TotalHits.Relation.EQUAL_TO;
import static org.apache.lucene.search.TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
Expand All @@ -24,7 +27,7 @@
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.OptionalLong;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.Weight.DefaultBulkScorer;
Expand All @@ -35,7 +38,7 @@ final class BooleanScorerSupplier extends ScorerSupplier {
private final ScoreMode scoreMode;
private final int minShouldMatch;
private final int maxDoc;
private long cost = -1;
private TotalHits estimatedCount = null;
private boolean topLevelScoringClause;

BooleanScorerSupplier(
Expand Down Expand Up @@ -69,21 +72,40 @@ final class BooleanScorerSupplier extends ScorerSupplier {
this.maxDoc = maxDoc;
}

private long computeCost() {
OptionalLong minRequiredCost =
private TotalHits computeCost(long upperBound) {

TotalHits minRequiredCost = null;
TotalHits totalHits = null;
for (ScorerSupplier scorerSupplier :
Stream.concat(subs.get(Occur.MUST).stream(), subs.get(Occur.FILTER).stream())
.mapToLong(ScorerSupplier::cost)
.min();
if (minRequiredCost.isPresent() && minShouldMatch == 0) {
return minRequiredCost.getAsLong();
.collect(Collectors.toList())) {
totalHits = scorerSupplier.isEstimatedPointCountGreaterThanOrEqualTo(upperBound);
if (totalHits.relation() == EQUAL_TO && totalHits.value() < upperBound) {
upperBound = totalHits.value();
minRequiredCost = totalHits;
} else if (minRequiredCost == null) {
minRequiredCost = totalHits;
}
}

if (minRequiredCost != null && minShouldMatch == 0) {
return minRequiredCost;
} else {
final Collection<ScorerSupplier> optionalScorers = subs.get(Occur.SHOULD);
final long shouldCost =
final TotalHits shouldCost =
ScorerUtil.costWithMinShouldMatch(
optionalScorers.stream().mapToLong(ScorerSupplier::cost),
optionalScorers.size(),
minShouldMatch);
return Math.min(minRequiredCost.orElse(Long.MAX_VALUE), shouldCost);
optionalScorers, optionalScorers.size(), minShouldMatch, upperBound);

if (shouldCost.relation() == EQUAL_TO) {
return shouldCost;
} else if (minRequiredCost != null && minRequiredCost.relation() == EQUAL_TO) {
return minRequiredCost;
} else if (minRequiredCost != null) {
// or we should return small one? it doesn't matter
return (shouldCost.value() > minRequiredCost.value() ? shouldCost : minRequiredCost);
} else {
return shouldCost;
}
}
}

Expand All @@ -103,10 +125,22 @@ public void setTopLevelScoringClause() throws IOException {

@Override
public long cost() {
if (cost == -1) {
cost = computeCost();
if (estimatedCount == null || estimatedCount.relation() == GREATER_THAN_OR_EQUAL_TO) {
estimatedCount = computeCost(Long.MAX_VALUE);
assert estimatedCount.value() >= 0;
}
return cost;
return estimatedCount.value();
}

@Override
public TotalHits isEstimatedPointCountGreaterThanOrEqualTo(long upperBound) {
if (estimatedCount == null
|| (estimatedCount.value() < upperBound
&& estimatedCount.relation() == GREATER_THAN_OR_EQUAL_TO)) {
estimatedCount = computeCost(upperBound);
assert estimatedCount.value() >= 0;
}
return estimatedCount;
}

@Override
Expand All @@ -126,7 +160,10 @@ public Scorer get(long leadCost) throws IOException {

private Scorer getInternal(long leadCost) throws IOException {
// three cases: conjunction, disjunction, or mix
leadCost = Math.min(leadCost, cost());
estimatedCount = isEstimatedPointCountGreaterThanOrEqualTo(leadCost);
if (estimatedCount.relation() == EQUAL_TO && estimatedCount.value() < leadCost) {
leadCost = estimatedCount.value();
}

// pure conjunction
if (subs.get(Occur.SHOULD).isEmpty()) {
Expand Down Expand Up @@ -202,10 +239,11 @@ BulkScorer booleanScorer() throws IOException {
// there will be no matches in the end) so we should only use
// BooleanScorer if matches are very dense
costThreshold = maxDoc / 3;
}

if (cost() < costThreshold) {
return null;
TotalHits estimatedCount = isEstimatedPointCountGreaterThanOrEqualTo(costThreshold);
if (estimatedCount.relation() == EQUAL_TO) {
return null;
}
}

positiveScorer = optionalBulkScorer();
Expand Down Expand Up @@ -315,10 +353,16 @@ private BulkScorer requiredBulkScorer() throws IOException {
return scorer;
}

long leadCost =
subs.get(Occur.MUST).stream().mapToLong(ScorerSupplier::cost).min().orElse(Long.MAX_VALUE);
leadCost =
subs.get(Occur.FILTER).stream().mapToLong(ScorerSupplier::cost).min().orElse(leadCost);
long leadCost = Long.MAX_VALUE;
TotalHits estimatedCount;
for (ScorerSupplier scorerSupplier :
Stream.concat(subs.get(Occur.MUST).stream(), subs.get(Occur.FILTER).stream())
.collect(Collectors.toList())) {
estimatedCount = scorerSupplier.isEstimatedPointCountGreaterThanOrEqualTo(leadCost);
if (estimatedCount.relation() == EQUAL_TO && estimatedCount.value() < leadCost) {
leadCost = estimatedCount.value();
}
}

List<Scorer> requiredNoScoring = new ArrayList<>();
for (ScorerSupplier ss : subs.get(Occur.FILTER)) {
Expand Down
Loading