Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Upgrade to lucene 8.1.0-snapshot-e460356abe #40952

Merged
merged 6 commits into from
Apr 16, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion buildSrc/version.properties
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
elasticsearch = 8.0.0
lucene = 8.0.0
lucene = 8.1.0-snapshot-e460356abe

bundled_jdk = 12+33

Expand Down
4 changes: 2 additions & 2 deletions docs/Versions.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ bare_version never includes -alpha or -beta
:bare_version: 8.0.0
:major-version: 8.x
:prev-major-version: 7.x
:lucene_version: 8.0.0
:lucene_version_path: 8_0_0
:lucene_version: 8.1.0
:lucene_version_path: 8_1_0
:branch: master
:jdk: 11.0.2
:jdk_major: 11
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,9 @@ public void testMultiTerms() throws IOException {

TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_word_delimiter");
String source = "PowerShot 500-42 wi-fi wi-fi-4000 j2se O'Neil's";
String[] expected = new String[] { "PowerShot", "PowerShot", "Power", "Shot", "50042",
"500-42", "500", "42", "wifi", "wi-fi", "wi", "fi", "wifi4000", "wi-fi-4000", "wi",
"fi", "4000", "j2se", "j2se", "j", "2", "se", "ONeil", "O'Neil's", "O", "Neil" };
String[] expected = new String[] { "PowerShot", "PowerShot", "Power", "Shot", "500-42",
"50042", "500", "42", "wi-fi", "wifi", "wi", "fi", "wi-fi-4000", "wifi4000", "wi",
"fi", "4000", "j2se", "j2se", "j", "2", "se", "O'Neil's", "ONeil", "O", "Neil" };
Tokenizer tokenizer = new WhitespaceTokenizer();
tokenizer.setReader(new StringReader(source));
int[] expectedIncr = new int[] { 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0,
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0a1addebde14147501b7d24a581a7a7288bc585d

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
b8b7d744e6294706b379ec7fdd2d6f1b6badc95b

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
c9dcc5568ccd4589f4a6871d2019661546106c83

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
bef6d901a9c8b4c6139de7419b3024e0c9fd4ad3

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
074c06d6f2254edae04acdd53bf6989a4343acc8

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
5cd2a341ab4524ec7ff40ba29faa4ead5e805413

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
ba55aba7d278f6201b4ebd6dafbc7edb6fe94f8c

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
543d99fd2ba4302f3555792236350b201514d821
1 change: 0 additions & 1 deletion server/licenses/lucene-analyzers-common-8.0.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
c20a8ae0c3bd769aa6c415ebea94ba466d9a631d
1 change: 0 additions & 1 deletion server/licenses/lucene-backward-codecs-8.0.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
6e8921ab37facdcc5c4b71f2612d72300d6de217
1 change: 0 additions & 1 deletion server/licenses/lucene-core-8.0.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
3e85f77d8f8ed1db53dba387fbdec55a9f912639
1 change: 0 additions & 1 deletion server/licenses/lucene-grouping-8.0.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
426a1822d888a6341f6bafccaad19e4a2ad88e25
1 change: 0 additions & 1 deletion server/licenses/lucene-highlighter-8.0.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
f83fa4b264198dfb12436a803309a60f5588481d
1 change: 0 additions & 1 deletion server/licenses/lucene-join-8.0.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
f381131abef51f77d26bccbb213d1c8563c19ec4
1 change: 0 additions & 1 deletion server/licenses/lucene-memory-8.0.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
8d8733551b9eb71e1f59688b8e78e0b481974d7a
1 change: 0 additions & 1 deletion server/licenses/lucene-misc-8.0.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
13da0b22f01dff4a01c9907425464a440695104b
1 change: 0 additions & 1 deletion server/licenses/lucene-queries-8.0.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
6c3de4dbb98b5cc00875d76e817929374bb9e710
1 change: 0 additions & 1 deletion server/licenses/lucene-queryparser-8.0.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
539ef199c74ae6891ac93f55632fe140b9d4c291
1 change: 0 additions & 1 deletion server/licenses/lucene-sandbox-8.0.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0371141f658e2157babd490f0a8ddbcd5114b371
1 change: 0 additions & 1 deletion server/licenses/lucene-spatial-8.0.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1bae56fbce29d6c597c00889dab1909f51f4aaac
1 change: 0 additions & 1 deletion server/licenses/lucene-spatial-extras-8.0.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
6eaed1dea9a18502ab9dffe55f081da6060373f7
1 change: 0 additions & 1 deletion server/licenses/lucene-spatial3d-8.0.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
e54c6be78275637544a3080874dd04b0d92755e5
1 change: 0 additions & 1 deletion server/licenses/lucene-suggest-8.0.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
e4c95d0bb740f18af520faebcebb968da3e0a687
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,10 @@

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.Term;
import org.apache.lucene.queries.CommonTermsQuery;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.PrefixQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper;
import org.apache.lucene.search.spans.SpanNearQuery;
import org.apache.lucene.search.spans.SpanOrQuery;
Expand All @@ -36,12 +34,9 @@
import org.apache.lucene.util.automaton.CharacterRunAutomaton;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery;
import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
import org.elasticsearch.index.search.ESToParentBlockJoinQuery;

import java.io.IOException;
import java.text.BreakIterator;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
Expand Down Expand Up @@ -151,11 +146,6 @@ protected FieldHighlighter getFieldHighlighter(String field, Query query, Set<Te
getScorer(field), maxPassages, (noMatchSize > 0 ? 1 : 0), getFormatter(field), noMatchSize, fieldValue);
}

@Override
protected Collection<Query> preMultiTermQueryRewrite(Query query) {
return rewriteCustomQuery(query);
}

@Override
protected Collection<Query> preSpanQueryRewrite(Query query) {
return rewriteCustomQuery(query);
Expand All @@ -175,7 +165,7 @@ private Collection<Query> rewriteCustomQuery(Query query) {
SpanQuery[] innerQueries = new SpanQuery[terms[i].length];
for (int j = 0; j < terms[i].length; j++) {
if (i == sizeMinus1) {
innerQueries[j] = new SpanMultiTermQueryWrapper<PrefixQuery>(new PrefixQuery(terms[i][j]));
innerQueries[j] = new SpanMultiTermQueryWrapper<>(new PrefixQuery(terms[i][j]));
} else {
innerQueries[j] = new SpanTermQuery(terms[i][j]);
}
Expand All @@ -200,17 +190,6 @@ private Collection<Query> rewriteCustomQuery(Query query) {
boolean inorder = (mpq.getSlop() == 0);
return Collections.singletonList(new SpanNearQuery(positionSpanQueries,
mpq.getSlop() + positionGaps, inorder));
} else if (query instanceof CommonTermsQuery) {
CommonTermsQuery ctq = (CommonTermsQuery) query;
List<Query> tqs = new ArrayList<> ();
for (Term term : ctq.getTerms()) {
tqs.add(new TermQuery(term));
}
return tqs;
} else if (query instanceof FunctionScoreQuery) {
return Collections.singletonList(((FunctionScoreQuery) query).getSubQuery());
} else if (query instanceof ESToParentBlockJoinQuery) {
return Collections.singletonList(((ESToParentBlockJoinQuery) query).getChildQuery());
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we not need these any more?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The unified highlighter uses a QueryVisitor to extract the terms now so I added a visit implementation for these queries since this code is only called to extract phrase and span queries:
https://github.com/elastic/elasticsearch/pull/40952/files#diff-4d092acdc70f570b9d18594dcffb95a6R80
https://github.com/elastic/elasticsearch/pull/40952/files#diff-80c91092d36bb5d49bdd3ef462f75e40R201

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍

} else {
return null;
}
Expand Down
2 changes: 1 addition & 1 deletion server/src/main/java/org/elasticsearch/Version.java
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
public static final int V_7_1_0_ID = 7010099;
public static final Version V_7_1_0 = new Version(V_7_1_0_ID, org.apache.lucene.util.Version.LUCENE_8_0_0);
public static final int V_8_0_0_ID = 8000099;
public static final Version V_8_0_0 = new Version(V_8_0_0_ID, org.apache.lucene.util.Version.LUCENE_8_0_0);
public static final Version V_8_0_0 = new Version(V_8_0_0_ID, org.apache.lucene.util.Version.LUCENE_8_1_0);
public static final Version CURRENT = V_8_0_0;


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,12 @@
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.FilterScorer;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryVisitor;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.ScorerSupplier;
import org.apache.lucene.search.Weight;
Expand Down Expand Up @@ -195,6 +197,12 @@ public CombineFunction getCombineFunction() {
return combineFunction;
}

@Override
public void visit(QueryVisitor visitor) {
// Highlighters must visit the child query to extract terms
subQuery.visit(visitor.getSubVisitor(BooleanClause.Occur.MUST, this));
}

@Override
public Query rewrite(IndexReader reader) throws IOException {
Query rewritten = super.rewrite(reader);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@

import org.apache.logging.log4j.Logger;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.FSTLoadMode;
import org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.Mode;
import org.apache.lucene.codecs.lucene80.Lucene80Codec;
import org.elasticsearch.common.Nullable;
Expand Down Expand Up @@ -48,7 +49,7 @@ public CodecService(@Nullable MapperService mapperService, Logger logger) {
final MapBuilder<String, Codec> codecs = MapBuilder.<String, Codec>newMapBuilder();
if (mapperService == null) {
codecs.put(DEFAULT_CODEC, new Lucene80Codec());
codecs.put(BEST_COMPRESSION_CODEC, new Lucene80Codec(Mode.BEST_COMPRESSION));
codecs.put(BEST_COMPRESSION_CODEC, new Lucene80Codec(Mode.BEST_COMPRESSION, FSTLoadMode.AUTO));
} else {
codecs.put(DEFAULT_CODEC,
new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.apache.logging.log4j.Logger;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.FSTLoadMode;
import org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat;
import org.apache.lucene.codecs.lucene80.Lucene80Codec;
import org.elasticsearch.common.lucene.Lucene;
Expand All @@ -47,7 +48,7 @@ public class PerFieldMappingPostingFormatCodec extends Lucene80Codec {
}

public PerFieldMappingPostingFormatCodec(Lucene50StoredFieldsFormat.Mode compressionMode, MapperService mapperService, Logger logger) {
super(compressionMode);
super(compressionMode, FSTLoadMode.AUTO);
this.mapperService = mapperService;
this.logger = logger;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,9 @@
import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.MatchesIterator;
import org.apache.lucene.search.QueryVisitor;
import org.apache.lucene.search.intervals.IntervalIterator;
import org.apache.lucene.search.intervals.Intervals;
import org.apache.lucene.search.intervals.IntervalsSource;
Expand All @@ -37,9 +37,10 @@

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Set;

/**
* Constructs an IntervalsSource based on analyzed text
Expand Down Expand Up @@ -287,14 +288,17 @@ public MatchesIterator matches(String field, LeafReaderContext ctx, int doc) {
return null;
}

@Override
public void visit(String field, QueryVisitor visitor) {}

@Override
public int minExtent() {
return 0;
}

@Override
public void extractTerms(String field, Set<Term> terms) {

public Collection<IntervalsSource> pullUpDisjunctions() {
return Collections.emptyList();
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,10 @@
package org.elasticsearch.index.search;

import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryVisitor;
import org.apache.lucene.search.Weight;
import org.apache.lucene.search.join.BitSetProducer;
import org.apache.lucene.search.join.ScoreMode;
Expand Down Expand Up @@ -74,6 +76,12 @@ public Query rewrite(IndexReader reader) throws IOException {
return super.rewrite(reader);
}

@Override
public void visit(QueryVisitor visitor) {
// Highlighters must visit the child query to extract terms
query.getChildQuery().visit(visitor.getSubVisitor(BooleanClause.Occur.MUST, this));
}

@Override
public Weight createWeight(IndexSearcher searcher, org.apache.lucene.search.ScoreMode scoreMode, float boost) throws IOException {
return query.createWeight(searcher, scoreMode, boost);
Expand Down
Loading