Update Apache Lucene to 9.6.0 (opensearch-project#7505)

Signed-off-by: Andriy Redko <[email protected]>
reta · May 10, 2023 · 789daba · 789daba
1 parent 3545959
commit 789daba
Show file tree

Hide file tree

Showing 63 changed files with 191 additions and 89 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -30,6 +30,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `org.apache.commons:commons-compress` from 1.22 to 1.23.0 (#7462)
 - Bump `com.azure:azure-core` from 1.34.0 to 1.39.0
 - Bump `com.networknt:json-schema-validator` from 1.0.78 to 1.0.81 (#7460)
+- Bump Apache Lucene to 9.6.0 ([#7505](https://github.com/opensearch-project/OpenSearch/pull/7505))
 
 ### Changed
 - Enable `./gradlew build` on MacOS by disabling bcw tests ([#7303](https://github.com/opensearch-project/OpenSearch/pull/7303))

diff --git a/buildSrc/version.properties b/buildSrc/version.properties
@@ -1,5 +1,5 @@
 opensearch        = 2.8.0
-lucene            = 9.5.0
+lucene            = 9.6.0
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 17.0.7+7

diff --git a/libs/core/licenses/lucene-core-9.5.0.jar.sha1 b/libs/core/licenses/lucene-core-9.5.0.jar.sha1
diff --git a/libs/core/licenses/lucene-core-9.6.0.jar.sha1 b/libs/core/licenses/lucene-core-9.6.0.jar.sha1
@@ -0,0 +1 @@
+2c08c7a491e9d033bb4806e0a45496e3a0667217
diff --git a/modules/lang-expression/licenses/lucene-expressions-9.5.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.5.0.jar.sha1
diff --git a/modules/lang-expression/licenses/lucene-expressions-9.6.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.6.0.jar.sha1
@@ -0,0 +1 @@
+f2b28bb17fa6a1415233b1db98bd6fd371afc9b3
diff --git a/...apper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldTypeTests.java b/...apper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldTypeTests.java
@@ -138,7 +138,10 @@ public void testPrefixQuery() {
 
         // this term should be too long to be rewriteable to a term query on the prefix field
         final String longTerm = "toolongforourprefixfieldthistermis";
-        assertThat(fieldType.prefixQuery(longTerm, CONSTANT_SCORE_REWRITE, MOCK_QSC), equalTo(new PrefixQuery(new Term(NAME, longTerm))));
+        assertThat(
+            fieldType.prefixQuery(longTerm, CONSTANT_SCORE_REWRITE, MOCK_QSC),
+            equalTo(new PrefixQuery(new Term(NAME, longTerm), CONSTANT_SCORE_REWRITE))
+        );
 
         OpenSearchException ee = expectThrows(
             OpenSearchException.class,

diff --git a/modules/parent-join/src/test/java/org/opensearch/join/query/HasChildQueryBuilderTests.java b/modules/parent-join/src/test/java/org/opensearch/join/query/HasChildQueryBuilderTests.java
@@ -36,7 +36,6 @@
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermInSetQuery;
@@ -50,9 +49,7 @@
 import org.opensearch.common.compress.CompressedXContent;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.xcontent.XContentBuilder;
-import org.opensearch.index.mapper.IdFieldMapper;
 import org.opensearch.index.mapper.MapperService;
-import org.opensearch.index.mapper.Uid;
 import org.opensearch.index.query.IdsQueryBuilder;
 import org.opensearch.index.query.InnerHitBuilder;
 import org.opensearch.index.query.InnerHitContextBuilder;
@@ -307,16 +304,9 @@ static void assertLateParsingQuery(Query query, String type, String id) throws I
         assertThat(booleanQuery.clauses().get(0).getQuery(), instanceOf(TermInSetQuery.class));
         TermInSetQuery termsQuery = (TermInSetQuery) booleanQuery.clauses().get(0).getQuery();
         Query rewrittenTermsQuery = termsQuery.rewrite(null);
-        assertThat(rewrittenTermsQuery, instanceOf(ConstantScoreQuery.class));
-        ConstantScoreQuery constantScoreQuery = (ConstantScoreQuery) rewrittenTermsQuery;
-        assertThat(constantScoreQuery.getQuery(), instanceOf(BooleanQuery.class));
-        BooleanQuery booleanTermsQuery = (BooleanQuery) constantScoreQuery.getQuery();
-        assertThat(booleanTermsQuery.clauses().toString(), booleanTermsQuery.clauses().size(), equalTo(1));
-        assertThat(booleanTermsQuery.clauses().get(0).getOccur(), equalTo(BooleanClause.Occur.SHOULD));
-        assertThat(booleanTermsQuery.clauses().get(0).getQuery(), instanceOf(TermQuery.class));
-        TermQuery termQuery = (TermQuery) booleanTermsQuery.clauses().get(0).getQuery();
-        assertThat(termQuery.getTerm().field(), equalTo(IdFieldMapper.NAME));
-        assertThat(termQuery.getTerm().bytes(), equalTo(Uid.encodeId(id)));
+        // The query is of type MultiTermQueryConstantScoreBlendedWrapper and is sealed inside Apache Lucene,
+        // no access to inner queries without using the reflection, falling back to stringified query comparison
+        assertThat(rewrittenTermsQuery.toString(), equalTo("_id:([ff 69 64])"));
         // check the type filter
         assertThat(booleanQuery.clauses().get(1).getOccur(), equalTo(BooleanClause.Occur.FILTER));
         assertEquals(new TermQuery(new Term("join_field", type)), booleanQuery.clauses().get(1).getQuery());

diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.5.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.5.0.jar.sha1
diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.6.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.6.0.jar.sha1
@@ -0,0 +1 @@
+560f0fcd2960aa8e4c2251b44ee81678545fb685
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.5.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.5.0.jar.sha1
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.6.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.6.0.jar.sha1
@@ -0,0 +1 @@
+839a51ef7acbb89e7d0e7a02a03bda080b7ded38
diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.5.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.5.0.jar.sha1
diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.6.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.6.0.jar.sha1
@@ -0,0 +1 @@
+f8a1b2e4c8dae00995aea5bf8f2ad47a1ab1f624
diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.5.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.5.0.jar.sha1
diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.6.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.6.0.jar.sha1
@@ -0,0 +1 @@
+47fc1c8246b2219769c3b28ee0af1c0c9b58922e
diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.5.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.5.0.jar.sha1
diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.6.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.6.0.jar.sha1
@@ -0,0 +1 @@
+7a31613d5d40276babb6ed927b498fd671f00c2d
diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.5.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.5.0.jar.sha1
diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.6.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.6.0.jar.sha1
@@ -0,0 +1 @@
+3b030f7348ff866f5ac0fd0b98407e89ae86413f
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.5.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.5.0.jar.sha1
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.6.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.6.0.jar.sha1
@@ -0,0 +1 @@
+8efe51c10d7b4bf22b6cd2e42c95b8665fb62038
diff --git a/server/licenses/lucene-analysis-common-9.5.0.jar.sha1 b/server/licenses/lucene-analysis-common-9.5.0.jar.sha1
diff --git a/server/licenses/lucene-analysis-common-9.6.0.jar.sha1 b/server/licenses/lucene-analysis-common-9.6.0.jar.sha1
@@ -0,0 +1 @@
+17a8b808bb7bf5f49fbfb9cbff821433e2908e22
diff --git a/server/licenses/lucene-backward-codecs-9.5.0.jar.sha1 b/server/licenses/lucene-backward-codecs-9.5.0.jar.sha1
diff --git a/server/licenses/lucene-backward-codecs-9.6.0.jar.sha1 b/server/licenses/lucene-backward-codecs-9.6.0.jar.sha1
@@ -0,0 +1 @@
+a4819ca127c46a2759fdf091f41512c56c441fa7
diff --git a/server/licenses/lucene-core-9.5.0.jar.sha1 b/server/licenses/lucene-core-9.5.0.jar.sha1
diff --git a/server/licenses/lucene-core-9.6.0.jar.sha1 b/server/licenses/lucene-core-9.6.0.jar.sha1
@@ -0,0 +1 @@
+2c08c7a491e9d033bb4806e0a45496e3a0667217
diff --git a/server/licenses/lucene-grouping-9.5.0.jar.sha1 b/server/licenses/lucene-grouping-9.5.0.jar.sha1
diff --git a/server/licenses/lucene-grouping-9.6.0.jar.sha1 b/server/licenses/lucene-grouping-9.6.0.jar.sha1
@@ -0,0 +1 @@
+b7c7d7c0ae0ae79b3dbdbb234c5fbe267d874412
diff --git a/server/licenses/lucene-highlighter-9.5.0.jar.sha1 b/server/licenses/lucene-highlighter-9.5.0.jar.sha1
diff --git a/server/licenses/lucene-highlighter-9.6.0.jar.sha1 b/server/licenses/lucene-highlighter-9.6.0.jar.sha1
@@ -0,0 +1 @@
+b9e14451c73fa0bc8e71a2cfe12a442f37a53c69
diff --git a/server/licenses/lucene-join-9.5.0.jar.sha1 b/server/licenses/lucene-join-9.5.0.jar.sha1
diff --git a/server/licenses/lucene-join-9.6.0.jar.sha1 b/server/licenses/lucene-join-9.6.0.jar.sha1
@@ -0,0 +1 @@
+6596a994b7b35b9f3cfdb76592b2067e9c0a9c60
diff --git a/server/licenses/lucene-memory-9.5.0.jar.sha1 b/server/licenses/lucene-memory-9.5.0.jar.sha1
diff --git a/server/licenses/lucene-memory-9.6.0.jar.sha1 b/server/licenses/lucene-memory-9.6.0.jar.sha1
@@ -0,0 +1 @@
+d7f9f0857463bd846abd73de2281135c511a44ba
diff --git a/server/licenses/lucene-misc-9.5.0.jar.sha1 b/server/licenses/lucene-misc-9.5.0.jar.sha1
diff --git a/server/licenses/lucene-misc-9.6.0.jar.sha1 b/server/licenses/lucene-misc-9.6.0.jar.sha1
@@ -0,0 +1 @@
+a8b498a061d63bfa4e91752219b41b2c654a8100
diff --git a/server/licenses/lucene-queries-9.5.0.jar.sha1 b/server/licenses/lucene-queries-9.5.0.jar.sha1
diff --git a/server/licenses/lucene-queries-9.6.0.jar.sha1 b/server/licenses/lucene-queries-9.6.0.jar.sha1
@@ -0,0 +1 @@
+aadf9fffd94006a397dbdb5ce49a3727ff7d8168
diff --git a/server/licenses/lucene-queryparser-9.5.0.jar.sha1 b/server/licenses/lucene-queryparser-9.5.0.jar.sha1
diff --git a/server/licenses/lucene-queryparser-9.6.0.jar.sha1 b/server/licenses/lucene-queryparser-9.6.0.jar.sha1
@@ -0,0 +1 @@
+c2f9ef9b7336981495272e99a1df678a97966102
diff --git a/server/licenses/lucene-sandbox-9.5.0.jar.sha1 b/server/licenses/lucene-sandbox-9.5.0.jar.sha1
diff --git a/server/licenses/lucene-sandbox-9.6.0.jar.sha1 b/server/licenses/lucene-sandbox-9.6.0.jar.sha1
@@ -0,0 +1 @@
+7d6a0f3ac5ced909a8d21a697f9241ddcbe662d9
diff --git a/server/licenses/lucene-spatial-extras-9.5.0.jar.sha1 b/server/licenses/lucene-spatial-extras-9.5.0.jar.sha1
diff --git a/server/licenses/lucene-spatial-extras-9.6.0.jar.sha1 b/server/licenses/lucene-spatial-extras-9.6.0.jar.sha1
@@ -0,0 +1 @@
+7df1a3ff8b099c6952d546712126d9db7c789e2c
diff --git a/server/licenses/lucene-spatial3d-9.5.0.jar.sha1 b/server/licenses/lucene-spatial3d-9.5.0.jar.sha1
diff --git a/server/licenses/lucene-spatial3d-9.6.0.jar.sha1 b/server/licenses/lucene-spatial3d-9.6.0.jar.sha1
@@ -0,0 +1 @@
+f00857ec2b769dc65b78933ce93fa3133804f38e
diff --git a/server/licenses/lucene-suggest-9.5.0.jar.sha1 b/server/licenses/lucene-suggest-9.5.0.jar.sha1
diff --git a/server/licenses/lucene-suggest-9.6.0.jar.sha1 b/server/licenses/lucene-suggest-9.6.0.jar.sha1
@@ -0,0 +1 @@
+269cd8d1cbe3ee2325b8fcaaf5f3c92eccc0cd14
diff --git a/server/src/main/java/org/opensearch/Version.java b/server/src/main/java/org/opensearch/Version.java
@@ -114,7 +114,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_2_7_0 = new Version(2070099, org.apache.lucene.util.Version.LUCENE_9_5_0);
     public static final Version V_2_7_1 = new Version(2070199, org.apache.lucene.util.Version.LUCENE_9_5_0);
 
-    public static final Version V_2_8_0 = new Version(2080099, org.apache.lucene.util.Version.LUCENE_9_5_0);
+    public static final Version V_2_8_0 = new Version(2080099, org.apache.lucene.util.Version.LUCENE_9_6_0);
     public static final Version CURRENT = V_2_8_0;
 
     public static Version readVersion(StreamInput in) throws IOException {

diff --git a/server/src/main/java/org/opensearch/index/mapper/BinaryFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/BinaryFieldMapper.java
@@ -32,7 +32,9 @@
 
 package org.opensearch.index.mapper;
 
+import org.apache.lucene.document.InvertableType;
 import org.apache.lucene.document.StoredField;
+import org.apache.lucene.document.StoredValue;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
 import org.opensearch.OpenSearchException;
@@ -273,5 +275,15 @@ public BytesRef binaryValue() {
             }
 
         }
+
+        @Override
+        public StoredValue storedValue() {
+            return null;
+        }
+
+        @Override
+        public InvertableType invertableType() {
+            return InvertableType.BINARY;
+        }
     }
 }
diff --git a/server/src/main/java/org/opensearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/RangeFieldMapper.java
@@ -32,6 +32,8 @@
 
 package org.opensearch.index.mapper;
 
+import org.apache.lucene.document.InvertableType;
+import org.apache.lucene.document.StoredValue;
 import org.apache.lucene.search.BoostQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
@@ -666,5 +668,15 @@ public BytesRef binaryValue() {
                 throw new OpenSearchException("failed to encode ranges", e);
             }
         }
+
+        @Override
+        public StoredValue storedValue() {
+            return null;
+        }
+
+        @Override
+        public InvertableType invertableType() {
+            return InvertableType.BINARY;
+        }
     }
 }
diff --git a/server/src/test/java/org/opensearch/deps/lucene/VectorHighlighterTests.java b/server/src/test/java/org/opensearch/deps/lucene/VectorHighlighterTests.java
@@ -111,7 +111,10 @@ public void testVectorHighlighterPrefixQuery() throws Exception {
         FastVectorHighlighter highlighter = new FastVectorHighlighter();
 
         PrefixQuery prefixQuery = new PrefixQuery(new Term("content", "ba"));
-        assertThat(prefixQuery.getRewriteMethod().getClass().getName(), equalTo(PrefixQuery.CONSTANT_SCORE_REWRITE.getClass().getName()));
+        assertThat(
+            prefixQuery.getRewriteMethod().getClass().getName(),
+            equalTo(PrefixQuery.CONSTANT_SCORE_BLENDED_REWRITE.getClass().getName())
+        );
         String fragment = highlighter.getBestFragment(
             highlighter.getFieldQuery(prefixQuery),
             reader,
@@ -128,7 +131,10 @@ public void testVectorHighlighterPrefixQuery() throws Exception {
 
         // now check with the custom field query
         prefixQuery = new PrefixQuery(new Term("content", "ba"));
-        assertThat(prefixQuery.getRewriteMethod().getClass().getName(), equalTo(PrefixQuery.CONSTANT_SCORE_REWRITE.getClass().getName()));
+        assertThat(
+            prefixQuery.getRewriteMethod().getClass().getName(),
+            equalTo(PrefixQuery.CONSTANT_SCORE_BLENDED_REWRITE.getClass().getName())
+        );
         fragment = highlighter.getBestFragment(
             new CustomFieldQuery(prefixQuery, reader, highlighter),
             reader,

diff --git a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java
@@ -40,6 +40,7 @@
 import org.apache.logging.log4j.core.appender.AbstractAppender;
 import org.apache.logging.log4j.core.filter.RegexFilter;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.KeywordField;
 import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.document.StoredField;
@@ -50,6 +51,7 @@
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.IndexableFieldType;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.LiveIndexWriterConfig;
@@ -121,6 +123,7 @@
 import org.opensearch.index.IndexSettings;
 import org.opensearch.index.VersionType;
 import org.opensearch.index.codec.CodecService;
+import org.opensearch.index.engine.Engine.IndexResult;
 import org.opensearch.index.fieldvisitor.FieldsVisitor;
 import org.opensearch.index.mapper.IdFieldMapper;
 import org.opensearch.index.mapper.ParseContext;
@@ -3945,27 +3948,28 @@ public void testHandleDocumentFailure() throws Exception {
                 assertNotNull(indexResult.getTranslogLocation());
                 engine.index(indexForDoc(doc2));
 
-                // test non document level failure is thrown
-                if (randomBoolean()) {
-                    // simulate close by corruption
-                    throwingIndexWriter.get().setThrowFailure(null);
-                    UncheckedIOException uncheckedIOException = expectThrows(UncheckedIOException.class, () -> {
+                try (engine) {
+                    // test non document level failure is thrown
+                    if (randomBoolean()) {
+                        // simulate close by corruption
+                        throwingIndexWriter.get().setThrowFailure(null);
                         Engine.Index index = indexForDoc(doc3);
-                        index.parsedDoc().rootDoc().add(new StoredField("foo", "bar") {
+                        index.parsedDoc().rootDoc().add(new KeywordField("foo", "bar", org.apache.lucene.document.Field.Store.YES) {
                             // this is a hack to add a failure during store document which triggers a tragic event
                             // and in turn fails the engine
                             @Override
                             public BytesRef binaryValue() {
                                 throw new UncheckedIOException(new MockDirectoryWrapper.FakeIOException());
                             }
                         });
-                        engine.index(index);
-                    });
-                    assertTrue(uncheckedIOException.getCause() instanceof MockDirectoryWrapper.FakeIOException);
-                } else {
-                    // normal close
-                    engine.close();
+
+                        final IndexResult r = engine.index(index);
+                        assertThat(r.isCreated(), equalTo(false));
+                        assertThat(r.getFailure(), instanceOf(UncheckedIOException.class));
+                        assertThat(r.getFailure().getCause(), instanceOf(MockDirectoryWrapper.FakeIOException.class));
+                    }
                 }
+
                 // now the engine is closed check we respond correctly
                 expectThrows(AlreadyClosedException.class, () -> engine.index(indexForDoc(doc1)));
                 expectThrows(AlreadyClosedException.class, () -> engine.delete(new Engine.Delete("", newUid(doc1), primaryTerm.get())));
@@ -3996,7 +4000,7 @@ public ParsedDocument newDeleteTombstoneDoc(String id) {
                         // this is a hack to add a failure during store document which triggers a tragic event
                         // and in turn fails the engine
                         @Override
-                        public BytesRef binaryValue() {
+                        public IndexableFieldType fieldType() {
                             throw tragicException;
                         }
                     });