Skip to content

Commit

Permalink
Update to Apache Lucene 9.6.0-snapshot-8a81515 (#6790)
Browse files Browse the repository at this point in the history
Signed-off-by: Andriy Redko <[email protected]>
  • Loading branch information
reta authored Apr 3, 2023
1 parent bcbb561 commit ac810fc
Show file tree
Hide file tree
Showing 60 changed files with 189 additions and 88 deletions.
2 changes: 1 addition & 1 deletion buildSrc/version.properties
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
opensearch = 3.0.0
lucene = 9.5.0
lucene = 9.6.0-snapshot-a3ae27f

bundled_jdk_vendor = adoptium
bundled_jdk = 19.0.2+7
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
d1d4772f25f73d7dd6fc37677e124d4ea7f96126
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,10 @@ public void testPrefixQuery() {

// this term should be too long to be rewriteable to a term query on the prefix field
final String longTerm = "toolongforourprefixfieldthistermis";
assertThat(fieldType.prefixQuery(longTerm, CONSTANT_SCORE_REWRITE, MOCK_QSC), equalTo(new PrefixQuery(new Term(NAME, longTerm))));
assertThat(
fieldType.prefixQuery(longTerm, CONSTANT_SCORE_REWRITE, MOCK_QSC),
equalTo(new PrefixQuery(new Term(NAME, longTerm), CONSTANT_SCORE_REWRITE))
);

OpenSearchException ee = expectThrows(
OpenSearchException.class,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.MatchNoDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermInSetQuery;
Expand All @@ -50,9 +49,7 @@
import org.opensearch.common.compress.CompressedXContent;
import org.opensearch.common.settings.Settings;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.index.mapper.IdFieldMapper;
import org.opensearch.index.mapper.MapperService;
import org.opensearch.index.mapper.Uid;
import org.opensearch.index.query.IdsQueryBuilder;
import org.opensearch.index.query.InnerHitBuilder;
import org.opensearch.index.query.InnerHitContextBuilder;
Expand Down Expand Up @@ -307,16 +304,9 @@ static void assertLateParsingQuery(Query query, String type, String id) throws I
assertThat(booleanQuery.clauses().get(0).getQuery(), instanceOf(TermInSetQuery.class));
TermInSetQuery termsQuery = (TermInSetQuery) booleanQuery.clauses().get(0).getQuery();
Query rewrittenTermsQuery = termsQuery.rewrite(null);
assertThat(rewrittenTermsQuery, instanceOf(ConstantScoreQuery.class));
ConstantScoreQuery constantScoreQuery = (ConstantScoreQuery) rewrittenTermsQuery;
assertThat(constantScoreQuery.getQuery(), instanceOf(BooleanQuery.class));
BooleanQuery booleanTermsQuery = (BooleanQuery) constantScoreQuery.getQuery();
assertThat(booleanTermsQuery.clauses().toString(), booleanTermsQuery.clauses().size(), equalTo(1));
assertThat(booleanTermsQuery.clauses().get(0).getOccur(), equalTo(BooleanClause.Occur.SHOULD));
assertThat(booleanTermsQuery.clauses().get(0).getQuery(), instanceOf(TermQuery.class));
TermQuery termQuery = (TermQuery) booleanTermsQuery.clauses().get(0).getQuery();
assertThat(termQuery.getTerm().field(), equalTo(IdFieldMapper.NAME));
assertThat(termQuery.getTerm().bytes(), equalTo(Uid.encodeId(id)));
// The query is of type MultiTermQueryConstantScoreBlendedWrapper and is sealed inside Apache Lucene,
// no access to inner queries without using the reflection, falling back to stringified query comparison
assertThat(rewrittenTermsQuery.toString(), equalTo("_id:([ff 69 64])"));
// check the type filter
assertThat(booleanQuery.clauses().get(1).getOccur(), equalTo(BooleanClause.Occur.FILTER));
assertEquals(new TermQuery(new Term("join_field", type)), booleanQuery.clauses().get(1).getQuery());
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
3a90a798de88ac1d4a8b0c081e087bc0fd05e8dd

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
dc5b2ec79dd5a88571bf9c371b6faed9e7a7462d

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
9688d5a5f5f942daeba6c43d50cc2bbe1c94c5c7

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
96726c96ab11ba53e7a283b1cb308920cef16ad0

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
ea7ae79fd9864ae04297c237fea7cd9224e42b5f

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
b5175ad05cf3f6c55750f63569b8b2ca8dd30fca

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
a79e5a8c0d2cf6dc8c76d05914c39df2cba69b01
1 change: 0 additions & 1 deletion server/licenses/lucene-analysis-common-9.5.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
af24672cb028edc12d3fb1d5cf07aff557042539
1 change: 0 additions & 1 deletion server/licenses/lucene-backward-codecs-9.5.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
7067e30e4374edc78ec3976787b6f1b9339c70ba
1 change: 0 additions & 1 deletion server/licenses/lucene-core-9.5.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
b72735dd63d0bd9c35534e8106d19e4df8738246
1 change: 0 additions & 1 deletion server/licenses/lucene-grouping-9.5.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
e20605f667703c8c0f11c42a39cd989b4449ba07
1 change: 0 additions & 1 deletion server/licenses/lucene-highlighter-9.5.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
aeb021e2a05353b75903da7f0d2c887e467d7050
1 change: 0 additions & 1 deletion server/licenses/lucene-join-9.5.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0f15970748550dfe1b030a493e9c6f195c7b8cbf
1 change: 0 additions & 1 deletion server/licenses/lucene-memory-9.5.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
714708d7e5566ea2e506d8df87066186befbb59f
1 change: 0 additions & 1 deletion server/licenses/lucene-misc-9.5.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
d5e017e7600bb6b04e3610274f1d115774e9c212
1 change: 0 additions & 1 deletion server/licenses/lucene-queries-9.5.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
8ff649c6e6c706ece75d66db711448cd858def22
1 change: 0 additions & 1 deletion server/licenses/lucene-queryparser-9.5.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
d9c88b97f94efdae9950cc00f3e2927ee1e6290e
1 change: 0 additions & 1 deletion server/licenses/lucene-sandbox-9.5.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
4c0094dc0d9b9e4affa6fd19572c2b02ad017bf0
1 change: 0 additions & 1 deletion server/licenses/lucene-spatial-extras-9.5.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
3a715552f9d21333766a4c2fbb6e7a202e79aac2
1 change: 0 additions & 1 deletion server/licenses/lucene-spatial3d-9.5.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
95cc1bd66ae35122828dd6a0a29f0747c117d51e
1 change: 0 additions & 1 deletion server/licenses/lucene-suggest-9.5.0.jar.sha1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
bc7faa1fbb9c8b3e5dd975942a7c85c82cbc4c3d
2 changes: 1 addition & 1 deletion server/src/main/java/org/opensearch/Version.java
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
public static final Version V_2_6_0 = new Version(2060099, org.apache.lucene.util.Version.LUCENE_9_5_0);
public static final Version V_2_6_1 = new Version(2060199, org.apache.lucene.util.Version.LUCENE_9_5_0);
public static final Version V_2_7_0 = new Version(2070099, org.apache.lucene.util.Version.LUCENE_9_5_0);
public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_5_0);
public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_6_0);
public static final Version CURRENT = V_3_0_0;

public static Version readVersion(StreamInput in) throws IOException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,9 @@

package org.opensearch.index.mapper;

import org.apache.lucene.document.InvertableType;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StoredValue;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.BytesRef;
import org.opensearch.OpenSearchException;
Expand Down Expand Up @@ -273,5 +275,15 @@ public BytesRef binaryValue() {
}

}

@Override
public StoredValue storedValue() {
return null;
}

@Override
public InvertableType invertableType() {
return InvertableType.BINARY;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@

package org.opensearch.index.mapper;

import org.apache.lucene.document.InvertableType;
import org.apache.lucene.document.StoredValue;
import org.apache.lucene.search.BoostQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.BytesRef;
Expand Down Expand Up @@ -660,5 +662,15 @@ public BytesRef binaryValue() {
throw new OpenSearchException("failed to encode ranges", e);
}
}

@Override
public StoredValue storedValue() {
return null;
}

@Override
public InvertableType invertableType() {
return InvertableType.BINARY;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,10 @@ public void testVectorHighlighterPrefixQuery() throws Exception {
FastVectorHighlighter highlighter = new FastVectorHighlighter();

PrefixQuery prefixQuery = new PrefixQuery(new Term("content", "ba"));
assertThat(prefixQuery.getRewriteMethod().getClass().getName(), equalTo(PrefixQuery.CONSTANT_SCORE_REWRITE.getClass().getName()));
assertThat(
prefixQuery.getRewriteMethod().getClass().getName(),
equalTo(PrefixQuery.CONSTANT_SCORE_BLENDED_REWRITE.getClass().getName())
);
String fragment = highlighter.getBestFragment(
highlighter.getFieldQuery(prefixQuery),
reader,
Expand All @@ -128,7 +131,10 @@ public void testVectorHighlighterPrefixQuery() throws Exception {

// now check with the custom field query
prefixQuery = new PrefixQuery(new Term("content", "ba"));
assertThat(prefixQuery.getRewriteMethod().getClass().getName(), equalTo(PrefixQuery.CONSTANT_SCORE_REWRITE.getClass().getName()));
assertThat(
prefixQuery.getRewriteMethod().getClass().getName(),
equalTo(PrefixQuery.CONSTANT_SCORE_BLENDED_REWRITE.getClass().getName())
);
fragment = highlighter.getBestFragment(
new CustomFieldQuery(prefixQuery, reader, highlighter),
reader,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
import org.apache.logging.log4j.core.appender.AbstractAppender;
import org.apache.logging.log4j.core.filter.RegexFilter;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.KeywordField;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.StoredField;
Expand All @@ -51,6 +52,7 @@
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.IndexableFieldType;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.LiveIndexWriterConfig;
Expand Down Expand Up @@ -122,6 +124,7 @@
import org.opensearch.index.IndexSettings;
import org.opensearch.index.VersionType;
import org.opensearch.index.codec.CodecService;
import org.opensearch.index.engine.Engine.IndexResult;
import org.opensearch.index.fieldvisitor.FieldsVisitor;
import org.opensearch.index.mapper.IdFieldMapper;
import org.opensearch.index.mapper.ParseContext;
Expand Down Expand Up @@ -3935,27 +3938,28 @@ public void testHandleDocumentFailure() throws Exception {
assertNotNull(indexResult.getTranslogLocation());
engine.index(indexForDoc(doc2));

// test non document level failure is thrown
if (randomBoolean()) {
// simulate close by corruption
throwingIndexWriter.get().setThrowFailure(null);
UncheckedIOException uncheckedIOException = expectThrows(UncheckedIOException.class, () -> {
try (engine) {
// test non document level failure is thrown
if (randomBoolean()) {
// simulate close by corruption
throwingIndexWriter.get().setThrowFailure(null);
Engine.Index index = indexForDoc(doc3);
index.parsedDoc().rootDoc().add(new StoredField("foo", "bar") {
index.parsedDoc().rootDoc().add(new KeywordField("foo", "bar", org.apache.lucene.document.Field.Store.YES) {
// this is a hack to add a failure during store document which triggers a tragic event
// and in turn fails the engine
@Override
public BytesRef binaryValue() {
throw new UncheckedIOException(new MockDirectoryWrapper.FakeIOException());
}
});
engine.index(index);
});
assertTrue(uncheckedIOException.getCause() instanceof MockDirectoryWrapper.FakeIOException);
} else {
// normal close
engine.close();

final IndexResult r = engine.index(index);
assertThat(r.isCreated(), equalTo(false));
assertThat(r.getFailure(), instanceOf(UncheckedIOException.class));
assertThat(r.getFailure().getCause(), instanceOf(MockDirectoryWrapper.FakeIOException.class));
}
}

// now the engine is closed check we respond correctly
expectThrows(AlreadyClosedException.class, () -> engine.index(indexForDoc(doc1)));
expectThrows(AlreadyClosedException.class, () -> engine.delete(new Engine.Delete("", newUid(doc1), primaryTerm.get())));
Expand Down Expand Up @@ -3986,7 +3990,7 @@ public ParsedDocument newDeleteTombstoneDoc(String id) {
// this is a hack to add a failure during store document which triggers a tragic event
// and in turn fails the engine
@Override
public BytesRef binaryValue() {
public IndexableFieldType fieldType() {
throw tragicException;
}
});
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,19 +33,22 @@
package org.opensearch.index.mapper;

import org.apache.lucene.index.Term;
import org.apache.lucene.search.MultiTermQuery;
import org.apache.lucene.search.PrefixQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.RegexpQuery;
import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.automaton.Operations;
import org.apache.lucene.util.automaton.RegExp;
import org.opensearch.OpenSearchException;

public class IgnoredFieldTypeTests extends FieldTypeTestCase {

public void testPrefixQuery() {
MappedFieldType ft = IgnoredFieldMapper.IgnoredFieldType.INSTANCE;

Query expected = new PrefixQuery(new Term("_ignored", new BytesRef("foo*")));
Query expected = new PrefixQuery(new Term("_ignored", new BytesRef("foo*")), MultiTermQuery.CONSTANT_SCORE_REWRITE);
assertEquals(expected, ft.prefixQuery("foo*", null, MOCK_QSC));

OpenSearchException ee = expectThrows(OpenSearchException.class, () -> ft.prefixQuery("foo*", null, MOCK_QSC_DISALLOW_EXPENSIVE));
Expand All @@ -59,7 +62,14 @@ public void testPrefixQuery() {
public void testRegexpQuery() {
MappedFieldType ft = IgnoredFieldMapper.IgnoredFieldType.INSTANCE;

Query expected = new RegexpQuery(new Term("_ignored", new BytesRef("foo?")));
Query expected = new RegexpQuery(
new Term("_ignored", new BytesRef("foo?")),
RegExp.ALL,
0,
RegexpQuery.DEFAULT_PROVIDER,
Operations.DEFAULT_DETERMINIZE_WORK_LIMIT,
MultiTermQuery.CONSTANT_SCORE_REWRITE
);
assertEquals(expected, ft.regexpQuery("foo?", 0, 0, 10, null, MOCK_QSC));

OpenSearchException ee = expectThrows(
Expand All @@ -73,7 +83,7 @@ public void testWildcardQuery() {
MappedFieldType ft = IgnoredFieldMapper.IgnoredFieldType.INSTANCE;

Query expected = new WildcardQuery(new Term("_ignored", new BytesRef("foo*")));
assertEquals(expected, ft.wildcardQuery("foo*", null, MOCK_QSC));
assertEquals(expected, ft.wildcardQuery("foo*", MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, MOCK_QSC));

OpenSearchException ee = expectThrows(
OpenSearchException.class,
Expand Down
Loading

0 comments on commit ac810fc

Please sign in to comment.