From 7ed1e6913fc0e057aae82266d86e91490e11f44d Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Thu, 11 May 2023 12:51:18 -0400 Subject: [PATCH] Update Apache Lucene to 9.6.0 (#7505) (#7510) Signed-off-by: Andriy Redko --- CHANGELOG.md | 1 + buildSrc/version.properties | 2 +- libs/core/licenses/lucene-core-9.5.0.jar.sha1 | 1 - libs/core/licenses/lucene-core-9.6.0.jar.sha1 | 1 + .../lucene-expressions-9.5.0.jar.sha1 | 1 - .../lucene-expressions-9.6.0.jar.sha1 | 1 + .../mapper/SearchAsYouTypeFieldTypeTests.java | 5 +- .../join/query/HasChildQueryBuilderTests.java | 16 ++---- .../lucene-analysis-icu-9.5.0.jar.sha1 | 1 - .../lucene-analysis-icu-9.6.0.jar.sha1 | 1 + .../lucene-analysis-kuromoji-9.5.0.jar.sha1 | 1 - .../lucene-analysis-kuromoji-9.6.0.jar.sha1 | 1 + .../lucene-analysis-nori-9.5.0.jar.sha1 | 1 - .../lucene-analysis-nori-9.6.0.jar.sha1 | 1 + .../lucene-analysis-phonetic-9.5.0.jar.sha1 | 1 - .../lucene-analysis-phonetic-9.6.0.jar.sha1 | 1 + .../lucene-analysis-smartcn-9.5.0.jar.sha1 | 1 - .../lucene-analysis-smartcn-9.6.0.jar.sha1 | 1 + .../lucene-analysis-stempel-9.5.0.jar.sha1 | 1 - .../lucene-analysis-stempel-9.6.0.jar.sha1 | 1 + .../lucene-analysis-morfologik-9.5.0.jar.sha1 | 1 - .../lucene-analysis-morfologik-9.6.0.jar.sha1 | 1 + .../lucene-analysis-common-9.5.0.jar.sha1 | 1 - .../lucene-analysis-common-9.6.0.jar.sha1 | 1 + .../lucene-backward-codecs-9.5.0.jar.sha1 | 1 - .../lucene-backward-codecs-9.6.0.jar.sha1 | 1 + server/licenses/lucene-core-9.5.0.jar.sha1 | 1 - server/licenses/lucene-core-9.6.0.jar.sha1 | 1 + .../licenses/lucene-grouping-9.5.0.jar.sha1 | 1 - .../licenses/lucene-grouping-9.6.0.jar.sha1 | 1 + .../lucene-highlighter-9.5.0.jar.sha1 | 1 - .../lucene-highlighter-9.6.0.jar.sha1 | 1 + server/licenses/lucene-join-9.5.0.jar.sha1 | 1 - server/licenses/lucene-join-9.6.0.jar.sha1 | 1 + server/licenses/lucene-memory-9.5.0.jar.sha1 | 1 - server/licenses/lucene-memory-9.6.0.jar.sha1 | 1 + server/licenses/lucene-misc-9.5.0.jar.sha1 | 1 - server/licenses/lucene-misc-9.6.0.jar.sha1 | 1 + server/licenses/lucene-queries-9.5.0.jar.sha1 | 1 - server/licenses/lucene-queries-9.6.0.jar.sha1 | 1 + .../lucene-queryparser-9.5.0.jar.sha1 | 1 - .../lucene-queryparser-9.6.0.jar.sha1 | 1 + server/licenses/lucene-sandbox-9.5.0.jar.sha1 | 1 - server/licenses/lucene-sandbox-9.6.0.jar.sha1 | 1 + .../lucene-spatial-extras-9.5.0.jar.sha1 | 1 - .../lucene-spatial-extras-9.6.0.jar.sha1 | 1 + .../licenses/lucene-spatial3d-9.5.0.jar.sha1 | 1 - .../licenses/lucene-spatial3d-9.6.0.jar.sha1 | 1 + server/licenses/lucene-suggest-9.5.0.jar.sha1 | 1 - server/licenses/lucene-suggest-9.6.0.jar.sha1 | 1 + .../src/main/java/org/opensearch/Version.java | 2 +- .../index/mapper/BinaryFieldMapper.java | 12 +++++ .../index/mapper/RangeFieldMapper.java | 12 +++++ .../deps/lucene/VectorHighlighterTests.java | 10 +++- .../index/engine/InternalEngineTests.java | 30 ++++++----- .../index/mapper/IgnoredFieldTypeTests.java | 16 ++++-- .../index/mapper/KeywordFieldTypeTests.java | 6 ++- .../index/mapper/RoutingFieldTypeTests.java | 7 +-- .../index/mapper/TextFieldTypeTests.java | 34 +++++++++--- .../index/query/DisMaxQueryBuilderTests.java | 6 ++- .../MatchBoolPrefixQueryBuilderTests.java | 7 +-- .../query/QueryStringQueryBuilderTests.java | 53 +++++++++++++++---- .../query/SimpleQueryStringBuilderTests.java | 15 ++++-- 63 files changed, 191 insertions(+), 89 deletions(-) delete mode 100644 libs/core/licenses/lucene-core-9.5.0.jar.sha1 create mode 100644 libs/core/licenses/lucene-core-9.6.0.jar.sha1 delete mode 100644 modules/lang-expression/licenses/lucene-expressions-9.5.0.jar.sha1 create mode 100644 modules/lang-expression/licenses/lucene-expressions-9.6.0.jar.sha1 delete mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.5.0.jar.sha1 create mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.6.0.jar.sha1 delete mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.5.0.jar.sha1 create mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.6.0.jar.sha1 delete mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.5.0.jar.sha1 create mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.6.0.jar.sha1 delete mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.5.0.jar.sha1 create mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.6.0.jar.sha1 delete mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.5.0.jar.sha1 create mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.6.0.jar.sha1 delete mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.5.0.jar.sha1 create mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.6.0.jar.sha1 delete mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.5.0.jar.sha1 create mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.6.0.jar.sha1 delete mode 100644 server/licenses/lucene-analysis-common-9.5.0.jar.sha1 create mode 100644 server/licenses/lucene-analysis-common-9.6.0.jar.sha1 delete mode 100644 server/licenses/lucene-backward-codecs-9.5.0.jar.sha1 create mode 100644 server/licenses/lucene-backward-codecs-9.6.0.jar.sha1 delete mode 100644 server/licenses/lucene-core-9.5.0.jar.sha1 create mode 100644 server/licenses/lucene-core-9.6.0.jar.sha1 delete mode 100644 server/licenses/lucene-grouping-9.5.0.jar.sha1 create mode 100644 server/licenses/lucene-grouping-9.6.0.jar.sha1 delete mode 100644 server/licenses/lucene-highlighter-9.5.0.jar.sha1 create mode 100644 server/licenses/lucene-highlighter-9.6.0.jar.sha1 delete mode 100644 server/licenses/lucene-join-9.5.0.jar.sha1 create mode 100644 server/licenses/lucene-join-9.6.0.jar.sha1 delete mode 100644 server/licenses/lucene-memory-9.5.0.jar.sha1 create mode 100644 server/licenses/lucene-memory-9.6.0.jar.sha1 delete mode 100644 server/licenses/lucene-misc-9.5.0.jar.sha1 create mode 100644 server/licenses/lucene-misc-9.6.0.jar.sha1 delete mode 100644 server/licenses/lucene-queries-9.5.0.jar.sha1 create mode 100644 server/licenses/lucene-queries-9.6.0.jar.sha1 delete mode 100644 server/licenses/lucene-queryparser-9.5.0.jar.sha1 create mode 100644 server/licenses/lucene-queryparser-9.6.0.jar.sha1 delete mode 100644 server/licenses/lucene-sandbox-9.5.0.jar.sha1 create mode 100644 server/licenses/lucene-sandbox-9.6.0.jar.sha1 delete mode 100644 server/licenses/lucene-spatial-extras-9.5.0.jar.sha1 create mode 100644 server/licenses/lucene-spatial-extras-9.6.0.jar.sha1 delete mode 100644 server/licenses/lucene-spatial3d-9.5.0.jar.sha1 create mode 100644 server/licenses/lucene-spatial3d-9.6.0.jar.sha1 delete mode 100644 server/licenses/lucene-suggest-9.5.0.jar.sha1 create mode 100644 server/licenses/lucene-suggest-9.6.0.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a65ca15f9371..02d91dbb67cac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Bump `org.apache.commons:commons-compress` from 1.22 to 1.23.0 (#7462) - Bump `com.azure:azure-core` from 1.34.0 to 1.39.0 - Bump `com.networknt:json-schema-validator` from 1.0.78 to 1.0.81 (#7460) +- Bump Apache Lucene to 9.6.0 ([#7505](https://github.com/opensearch-project/OpenSearch/pull/7505)) ### Changed - Enable `./gradlew build` on MacOS by disabling bcw tests ([#7303](https://github.com/opensearch-project/OpenSearch/pull/7303)) diff --git a/buildSrc/version.properties b/buildSrc/version.properties index da205b1f939c8..12672f917271c 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -1,5 +1,5 @@ opensearch = 2.8.0 -lucene = 9.5.0 +lucene = 9.6.0 bundled_jdk_vendor = adoptium bundled_jdk = 17.0.7+7 diff --git a/libs/core/licenses/lucene-core-9.5.0.jar.sha1 b/libs/core/licenses/lucene-core-9.5.0.jar.sha1 deleted file mode 100644 index bca72d4375dab..0000000000000 --- a/libs/core/licenses/lucene-core-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -bba4ba5d30e71a5f0017e45e8469db8cff8ad102 \ No newline at end of file diff --git a/libs/core/licenses/lucene-core-9.6.0.jar.sha1 b/libs/core/licenses/lucene-core-9.6.0.jar.sha1 new file mode 100644 index 0000000000000..b467b1754fd9d --- /dev/null +++ b/libs/core/licenses/lucene-core-9.6.0.jar.sha1 @@ -0,0 +1 @@ +2c08c7a491e9d033bb4806e0a45496e3a0667217 \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-9.5.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.5.0.jar.sha1 deleted file mode 100644 index c1340a55815e5..0000000000000 --- a/modules/lang-expression/licenses/lucene-expressions-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -62f3d3630ecc14069d5c24b9693df5a2787f8202 \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-9.6.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.6.0.jar.sha1 new file mode 100644 index 0000000000000..3b5e06a6eebf4 --- /dev/null +++ b/modules/lang-expression/licenses/lucene-expressions-9.6.0.jar.sha1 @@ -0,0 +1 @@ +f2b28bb17fa6a1415233b1db98bd6fd371afc9b3 \ No newline at end of file diff --git a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldTypeTests.java index 77e34560d4cad..3b70455960966 100644 --- a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldTypeTests.java @@ -138,7 +138,10 @@ public void testPrefixQuery() { // this term should be too long to be rewriteable to a term query on the prefix field final String longTerm = "toolongforourprefixfieldthistermis"; - assertThat(fieldType.prefixQuery(longTerm, CONSTANT_SCORE_REWRITE, MOCK_QSC), equalTo(new PrefixQuery(new Term(NAME, longTerm)))); + assertThat( + fieldType.prefixQuery(longTerm, CONSTANT_SCORE_REWRITE, MOCK_QSC), + equalTo(new PrefixQuery(new Term(NAME, longTerm), CONSTANT_SCORE_REWRITE)) + ); OpenSearchException ee = expectThrows( OpenSearchException.class, diff --git a/modules/parent-join/src/test/java/org/opensearch/join/query/HasChildQueryBuilderTests.java b/modules/parent-join/src/test/java/org/opensearch/join/query/HasChildQueryBuilderTests.java index e7f485c765cb8..4825dd0b26ac6 100644 --- a/modules/parent-join/src/test/java/org/opensearch/join/query/HasChildQueryBuilderTests.java +++ b/modules/parent-join/src/test/java/org/opensearch/join/query/HasChildQueryBuilderTests.java @@ -36,7 +36,6 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermInSetQuery; @@ -50,9 +49,7 @@ import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.settings.Settings; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.index.mapper.IdFieldMapper; import org.opensearch.index.mapper.MapperService; -import org.opensearch.index.mapper.Uid; import org.opensearch.index.query.IdsQueryBuilder; import org.opensearch.index.query.InnerHitBuilder; import org.opensearch.index.query.InnerHitContextBuilder; @@ -307,16 +304,9 @@ static void assertLateParsingQuery(Query query, String type, String id) throws I assertThat(booleanQuery.clauses().get(0).getQuery(), instanceOf(TermInSetQuery.class)); TermInSetQuery termsQuery = (TermInSetQuery) booleanQuery.clauses().get(0).getQuery(); Query rewrittenTermsQuery = termsQuery.rewrite(null); - assertThat(rewrittenTermsQuery, instanceOf(ConstantScoreQuery.class)); - ConstantScoreQuery constantScoreQuery = (ConstantScoreQuery) rewrittenTermsQuery; - assertThat(constantScoreQuery.getQuery(), instanceOf(BooleanQuery.class)); - BooleanQuery booleanTermsQuery = (BooleanQuery) constantScoreQuery.getQuery(); - assertThat(booleanTermsQuery.clauses().toString(), booleanTermsQuery.clauses().size(), equalTo(1)); - assertThat(booleanTermsQuery.clauses().get(0).getOccur(), equalTo(BooleanClause.Occur.SHOULD)); - assertThat(booleanTermsQuery.clauses().get(0).getQuery(), instanceOf(TermQuery.class)); - TermQuery termQuery = (TermQuery) booleanTermsQuery.clauses().get(0).getQuery(); - assertThat(termQuery.getTerm().field(), equalTo(IdFieldMapper.NAME)); - assertThat(termQuery.getTerm().bytes(), equalTo(Uid.encodeId(id))); + // The query is of type MultiTermQueryConstantScoreBlendedWrapper and is sealed inside Apache Lucene, + // no access to inner queries without using the reflection, falling back to stringified query comparison + assertThat(rewrittenTermsQuery.toString(), equalTo("_id:([ff 69 64])")); // check the type filter assertThat(booleanQuery.clauses().get(1).getOccur(), equalTo(BooleanClause.Occur.FILTER)); assertEquals(new TermQuery(new Term("join_field", type)), booleanQuery.clauses().get(1).getQuery()); diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.5.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.5.0.jar.sha1 deleted file mode 100644 index 1d3eb41aab0a1..0000000000000 --- a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -11c816250e4ff106151fd8cb69e61ead4fb4a8dd \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.6.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.6.0.jar.sha1 new file mode 100644 index 0000000000000..df2381354a704 --- /dev/null +++ b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.6.0.jar.sha1 @@ -0,0 +1 @@ +560f0fcd2960aa8e4c2251b44ee81678545fb685 \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.5.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.5.0.jar.sha1 deleted file mode 100644 index 31f1eb3991d5a..0000000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4555fc64ce9d63c6c1f4983e816526f896606e9f \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.6.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.6.0.jar.sha1 new file mode 100644 index 0000000000000..0c70fc62e0a62 --- /dev/null +++ b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.6.0.jar.sha1 @@ -0,0 +1 @@ +839a51ef7acbb89e7d0e7a02a03bda080b7ded38 \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.5.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.5.0.jar.sha1 deleted file mode 100644 index 47bfe0e4b667d..0000000000000 --- a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2fd55607da8adf8234169d9f4120119c05a8175c \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.6.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.6.0.jar.sha1 new file mode 100644 index 0000000000000..4970e81334308 --- /dev/null +++ b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.6.0.jar.sha1 @@ -0,0 +1 @@ +f8a1b2e4c8dae00995aea5bf8f2ad47a1ab1f624 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.5.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.5.0.jar.sha1 deleted file mode 100644 index 99ce9a0a4cb9a..0000000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c97163035f1d46249505d15fbdc427b6ae03549a \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.6.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.6.0.jar.sha1 new file mode 100644 index 0000000000000..70cccbef4e7ae --- /dev/null +++ b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.6.0.jar.sha1 @@ -0,0 +1 @@ +47fc1c8246b2219769c3b28ee0af1c0c9b58922e \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.5.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.5.0.jar.sha1 deleted file mode 100644 index e63db9c407a77..0000000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a08ae3e2212837c98d5e1174cf1a293a5c8c0373 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.6.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.6.0.jar.sha1 new file mode 100644 index 0000000000000..52224968bbf10 --- /dev/null +++ b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.6.0.jar.sha1 @@ -0,0 +1 @@ +7a31613d5d40276babb6ed927b498fd671f00c2d \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.5.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.5.0.jar.sha1 deleted file mode 100644 index d2f956a296f96..0000000000000 --- a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -483906d2ebad0ff910d52382c2f027b511788b58 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.6.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.6.0.jar.sha1 new file mode 100644 index 0000000000000..a40945b946375 --- /dev/null +++ b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.6.0.jar.sha1 @@ -0,0 +1 @@ +3b030f7348ff866f5ac0fd0b98407e89ae86413f \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.5.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.5.0.jar.sha1 deleted file mode 100644 index d28a2fa06fe28..0000000000000 --- a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -cb00cd193e2f927e6357cf33e0880034571ea528 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.6.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.6.0.jar.sha1 new file mode 100644 index 0000000000000..64e250dba3441 --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.6.0.jar.sha1 @@ -0,0 +1 @@ +8efe51c10d7b4bf22b6cd2e42c95b8665fb62038 \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.5.0.jar.sha1 b/server/licenses/lucene-analysis-common-9.5.0.jar.sha1 deleted file mode 100644 index bcd78036b0c99..0000000000000 --- a/server/licenses/lucene-analysis-common-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f68660102455a466f98cac0501723bed7e7c6407 \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.6.0.jar.sha1 b/server/licenses/lucene-analysis-common-9.6.0.jar.sha1 new file mode 100644 index 0000000000000..c8de75f3ea24a --- /dev/null +++ b/server/licenses/lucene-analysis-common-9.6.0.jar.sha1 @@ -0,0 +1 @@ +17a8b808bb7bf5f49fbfb9cbff821433e2908e22 \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.5.0.jar.sha1 b/server/licenses/lucene-backward-codecs-9.5.0.jar.sha1 deleted file mode 100644 index 4000ee7fc0e93..0000000000000 --- a/server/licenses/lucene-backward-codecs-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -66be22239d5058b0b8a8aeba03dc047a276efafa \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.6.0.jar.sha1 b/server/licenses/lucene-backward-codecs-9.6.0.jar.sha1 new file mode 100644 index 0000000000000..402678737f498 --- /dev/null +++ b/server/licenses/lucene-backward-codecs-9.6.0.jar.sha1 @@ -0,0 +1 @@ +a4819ca127c46a2759fdf091f41512c56c441fa7 \ No newline at end of file diff --git a/server/licenses/lucene-core-9.5.0.jar.sha1 b/server/licenses/lucene-core-9.5.0.jar.sha1 deleted file mode 100644 index bca72d4375dab..0000000000000 --- a/server/licenses/lucene-core-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -bba4ba5d30e71a5f0017e45e8469db8cff8ad102 \ No newline at end of file diff --git a/server/licenses/lucene-core-9.6.0.jar.sha1 b/server/licenses/lucene-core-9.6.0.jar.sha1 new file mode 100644 index 0000000000000..b467b1754fd9d --- /dev/null +++ b/server/licenses/lucene-core-9.6.0.jar.sha1 @@ -0,0 +1 @@ +2c08c7a491e9d033bb4806e0a45496e3a0667217 \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.5.0.jar.sha1 b/server/licenses/lucene-grouping-9.5.0.jar.sha1 deleted file mode 100644 index 08ed52d3379ec..0000000000000 --- a/server/licenses/lucene-grouping-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b15fe0a55a82168c810d0447ec2e244d16d94f01 \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.6.0.jar.sha1 b/server/licenses/lucene-grouping-9.6.0.jar.sha1 new file mode 100644 index 0000000000000..3bd407911646e --- /dev/null +++ b/server/licenses/lucene-grouping-9.6.0.jar.sha1 @@ -0,0 +1 @@ +b7c7d7c0ae0ae79b3dbdbb234c5fbe267d874412 \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.5.0.jar.sha1 b/server/licenses/lucene-highlighter-9.5.0.jar.sha1 deleted file mode 100644 index 0df6615e43c66..0000000000000 --- a/server/licenses/lucene-highlighter-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8b5e3cea3370838bda1d730cf55176c24a763d2e \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.6.0.jar.sha1 b/server/licenses/lucene-highlighter-9.6.0.jar.sha1 new file mode 100644 index 0000000000000..f21c0ab19eb21 --- /dev/null +++ b/server/licenses/lucene-highlighter-9.6.0.jar.sha1 @@ -0,0 +1 @@ +b9e14451c73fa0bc8e71a2cfe12a442f37a53c69 \ No newline at end of file diff --git a/server/licenses/lucene-join-9.5.0.jar.sha1 b/server/licenses/lucene-join-9.5.0.jar.sha1 deleted file mode 100644 index d441e3215820a..0000000000000 --- a/server/licenses/lucene-join-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -85a4208614a5660297effce441883687b010073b \ No newline at end of file diff --git a/server/licenses/lucene-join-9.6.0.jar.sha1 b/server/licenses/lucene-join-9.6.0.jar.sha1 new file mode 100644 index 0000000000000..626557fe357ae --- /dev/null +++ b/server/licenses/lucene-join-9.6.0.jar.sha1 @@ -0,0 +1 @@ +6596a994b7b35b9f3cfdb76592b2067e9c0a9c60 \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.5.0.jar.sha1 b/server/licenses/lucene-memory-9.5.0.jar.sha1 deleted file mode 100644 index 0fb289af21987..0000000000000 --- a/server/licenses/lucene-memory-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d632e63d08837be715046c8ccb4fb804acd3d7e4 \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.6.0.jar.sha1 b/server/licenses/lucene-memory-9.6.0.jar.sha1 new file mode 100644 index 0000000000000..815fa58383190 --- /dev/null +++ b/server/licenses/lucene-memory-9.6.0.jar.sha1 @@ -0,0 +1 @@ +d7f9f0857463bd846abd73de2281135c511a44ba \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.5.0.jar.sha1 b/server/licenses/lucene-misc-9.5.0.jar.sha1 deleted file mode 100644 index 985fbb18ea6d0..0000000000000 --- a/server/licenses/lucene-misc-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -64773801b8ba8141f4256d22da598de40d6f3033 \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.6.0.jar.sha1 b/server/licenses/lucene-misc-9.6.0.jar.sha1 new file mode 100644 index 0000000000000..19163be33645e --- /dev/null +++ b/server/licenses/lucene-misc-9.6.0.jar.sha1 @@ -0,0 +1 @@ +a8b498a061d63bfa4e91752219b41b2c654a8100 \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.5.0.jar.sha1 b/server/licenses/lucene-queries-9.5.0.jar.sha1 deleted file mode 100644 index 4ac6d8161fc58..0000000000000 --- a/server/licenses/lucene-queries-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ae7930fa0ea91198905d695a222ef7f09de1c2dd \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.6.0.jar.sha1 b/server/licenses/lucene-queries-9.6.0.jar.sha1 new file mode 100644 index 0000000000000..4c9f4f5a4806c --- /dev/null +++ b/server/licenses/lucene-queries-9.6.0.jar.sha1 @@ -0,0 +1 @@ +aadf9fffd94006a397dbdb5ce49a3727ff7d8168 \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.5.0.jar.sha1 b/server/licenses/lucene-queryparser-9.5.0.jar.sha1 deleted file mode 100644 index 45b6229e713c2..0000000000000 --- a/server/licenses/lucene-queryparser-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b3e3d9434bcfce242e242364c64eab0888d285e2 \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.6.0.jar.sha1 b/server/licenses/lucene-queryparser-9.6.0.jar.sha1 new file mode 100644 index 0000000000000..a4ed68160aae9 --- /dev/null +++ b/server/licenses/lucene-queryparser-9.6.0.jar.sha1 @@ -0,0 +1 @@ +c2f9ef9b7336981495272e99a1df678a97966102 \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.5.0.jar.sha1 b/server/licenses/lucene-sandbox-9.5.0.jar.sha1 deleted file mode 100644 index d43ebd2d27119..0000000000000 --- a/server/licenses/lucene-sandbox-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -408be287b0f421cf8afd655579a0cc65aba61b28 \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.6.0.jar.sha1 b/server/licenses/lucene-sandbox-9.6.0.jar.sha1 new file mode 100644 index 0000000000000..a343fecc87040 --- /dev/null +++ b/server/licenses/lucene-sandbox-9.6.0.jar.sha1 @@ -0,0 +1 @@ +7d6a0f3ac5ced909a8d21a697f9241ddcbe662d9 \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.5.0.jar.sha1 b/server/licenses/lucene-spatial-extras-9.5.0.jar.sha1 deleted file mode 100644 index b44f49f3601ba..0000000000000 --- a/server/licenses/lucene-spatial-extras-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4960c09365d20f024bf668f60d7c3b4e54f03750 \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.6.0.jar.sha1 b/server/licenses/lucene-spatial-extras-9.6.0.jar.sha1 new file mode 100644 index 0000000000000..78de0056a41bc --- /dev/null +++ b/server/licenses/lucene-spatial-extras-9.6.0.jar.sha1 @@ -0,0 +1 @@ +7df1a3ff8b099c6952d546712126d9db7c789e2c \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.5.0.jar.sha1 b/server/licenses/lucene-spatial3d-9.5.0.jar.sha1 deleted file mode 100644 index 369e569ee754f..0000000000000 --- a/server/licenses/lucene-spatial3d-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -01ed55dd384e654076b76d7083feee6ac3efe02b \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.6.0.jar.sha1 b/server/licenses/lucene-spatial3d-9.6.0.jar.sha1 new file mode 100644 index 0000000000000..25382b893450b --- /dev/null +++ b/server/licenses/lucene-spatial3d-9.6.0.jar.sha1 @@ -0,0 +1 @@ +f00857ec2b769dc65b78933ce93fa3133804f38e \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.5.0.jar.sha1 b/server/licenses/lucene-suggest-9.5.0.jar.sha1 deleted file mode 100644 index c44ac0ec028f4..0000000000000 --- a/server/licenses/lucene-suggest-9.5.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -195624eb3a09b3dc37cc5fa51ae2f7bd61553985 \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.6.0.jar.sha1 b/server/licenses/lucene-suggest-9.6.0.jar.sha1 new file mode 100644 index 0000000000000..f16767a838f63 --- /dev/null +++ b/server/licenses/lucene-suggest-9.6.0.jar.sha1 @@ -0,0 +1 @@ +269cd8d1cbe3ee2325b8fcaaf5f3c92eccc0cd14 \ No newline at end of file diff --git a/server/src/main/java/org/opensearch/Version.java b/server/src/main/java/org/opensearch/Version.java index 495456fc4f0bc..7d9ab63b3252c 100644 --- a/server/src/main/java/org/opensearch/Version.java +++ b/server/src/main/java/org/opensearch/Version.java @@ -114,7 +114,7 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_2_7_0 = new Version(2070099, org.apache.lucene.util.Version.LUCENE_9_5_0); public static final Version V_2_7_1 = new Version(2070199, org.apache.lucene.util.Version.LUCENE_9_5_0); - public static final Version V_2_8_0 = new Version(2080099, org.apache.lucene.util.Version.LUCENE_9_5_0); + public static final Version V_2_8_0 = new Version(2080099, org.apache.lucene.util.Version.LUCENE_9_6_0); public static final Version CURRENT = V_2_8_0; public static Version readVersion(StreamInput in) throws IOException { diff --git a/server/src/main/java/org/opensearch/index/mapper/BinaryFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/BinaryFieldMapper.java index 3977c7c1512fa..64f9a7e6f09cd 100644 --- a/server/src/main/java/org/opensearch/index/mapper/BinaryFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/BinaryFieldMapper.java @@ -32,7 +32,9 @@ package org.opensearch.index.mapper; +import org.apache.lucene.document.InvertableType; import org.apache.lucene.document.StoredField; +import org.apache.lucene.document.StoredValue; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.opensearch.OpenSearchException; @@ -273,5 +275,15 @@ public BytesRef binaryValue() { } } + + @Override + public StoredValue storedValue() { + return null; + } + + @Override + public InvertableType invertableType() { + return InvertableType.BINARY; + } } } diff --git a/server/src/main/java/org/opensearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/RangeFieldMapper.java index 3fc8d4399db53..2c55dd8978bf3 100644 --- a/server/src/main/java/org/opensearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/RangeFieldMapper.java @@ -32,6 +32,8 @@ package org.opensearch.index.mapper; +import org.apache.lucene.document.InvertableType; +import org.apache.lucene.document.StoredValue; import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; @@ -666,5 +668,15 @@ public BytesRef binaryValue() { throw new OpenSearchException("failed to encode ranges", e); } } + + @Override + public StoredValue storedValue() { + return null; + } + + @Override + public InvertableType invertableType() { + return InvertableType.BINARY; + } } } diff --git a/server/src/test/java/org/opensearch/deps/lucene/VectorHighlighterTests.java b/server/src/test/java/org/opensearch/deps/lucene/VectorHighlighterTests.java index e91da4f5ee46e..0f831ff28af79 100644 --- a/server/src/test/java/org/opensearch/deps/lucene/VectorHighlighterTests.java +++ b/server/src/test/java/org/opensearch/deps/lucene/VectorHighlighterTests.java @@ -111,7 +111,10 @@ public void testVectorHighlighterPrefixQuery() throws Exception { FastVectorHighlighter highlighter = new FastVectorHighlighter(); PrefixQuery prefixQuery = new PrefixQuery(new Term("content", "ba")); - assertThat(prefixQuery.getRewriteMethod().getClass().getName(), equalTo(PrefixQuery.CONSTANT_SCORE_REWRITE.getClass().getName())); + assertThat( + prefixQuery.getRewriteMethod().getClass().getName(), + equalTo(PrefixQuery.CONSTANT_SCORE_BLENDED_REWRITE.getClass().getName()) + ); String fragment = highlighter.getBestFragment( highlighter.getFieldQuery(prefixQuery), reader, @@ -128,7 +131,10 @@ public void testVectorHighlighterPrefixQuery() throws Exception { // now check with the custom field query prefixQuery = new PrefixQuery(new Term("content", "ba")); - assertThat(prefixQuery.getRewriteMethod().getClass().getName(), equalTo(PrefixQuery.CONSTANT_SCORE_REWRITE.getClass().getName())); + assertThat( + prefixQuery.getRewriteMethod().getClass().getName(), + equalTo(PrefixQuery.CONSTANT_SCORE_BLENDED_REWRITE.getClass().getName()) + ); fragment = highlighter.getBestFragment( new CustomFieldQuery(prefixQuery, reader, highlighter), reader, diff --git a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java index 1154e7a28fb3f..8f1570ffb3d36 100644 --- a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java @@ -40,6 +40,7 @@ import org.apache.logging.log4j.core.appender.AbstractAppender; import org.apache.logging.log4j.core.filter.RegexFilter; import org.apache.lucene.document.Field; +import org.apache.lucene.document.KeywordField; import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.StoredField; @@ -50,6 +51,7 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.IndexableFieldType; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.LiveIndexWriterConfig; @@ -121,6 +123,7 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.VersionType; import org.opensearch.index.codec.CodecService; +import org.opensearch.index.engine.Engine.IndexResult; import org.opensearch.index.fieldvisitor.FieldsVisitor; import org.opensearch.index.mapper.IdFieldMapper; import org.opensearch.index.mapper.ParseContext; @@ -3945,13 +3948,13 @@ public void testHandleDocumentFailure() throws Exception { assertNotNull(indexResult.getTranslogLocation()); engine.index(indexForDoc(doc2)); - // test non document level failure is thrown - if (randomBoolean()) { - // simulate close by corruption - throwingIndexWriter.get().setThrowFailure(null); - UncheckedIOException uncheckedIOException = expectThrows(UncheckedIOException.class, () -> { + try (engine) { + // test non document level failure is thrown + if (randomBoolean()) { + // simulate close by corruption + throwingIndexWriter.get().setThrowFailure(null); Engine.Index index = indexForDoc(doc3); - index.parsedDoc().rootDoc().add(new StoredField("foo", "bar") { + index.parsedDoc().rootDoc().add(new KeywordField("foo", "bar", org.apache.lucene.document.Field.Store.YES) { // this is a hack to add a failure during store document which triggers a tragic event // and in turn fails the engine @Override @@ -3959,13 +3962,14 @@ public BytesRef binaryValue() { throw new UncheckedIOException(new MockDirectoryWrapper.FakeIOException()); } }); - engine.index(index); - }); - assertTrue(uncheckedIOException.getCause() instanceof MockDirectoryWrapper.FakeIOException); - } else { - // normal close - engine.close(); + + final IndexResult r = engine.index(index); + assertThat(r.isCreated(), equalTo(false)); + assertThat(r.getFailure(), instanceOf(UncheckedIOException.class)); + assertThat(r.getFailure().getCause(), instanceOf(MockDirectoryWrapper.FakeIOException.class)); + } } + // now the engine is closed check we respond correctly expectThrows(AlreadyClosedException.class, () -> engine.index(indexForDoc(doc1))); expectThrows(AlreadyClosedException.class, () -> engine.delete(new Engine.Delete("", newUid(doc1), primaryTerm.get()))); @@ -3996,7 +4000,7 @@ public ParsedDocument newDeleteTombstoneDoc(String id) { // this is a hack to add a failure during store document which triggers a tragic event // and in turn fails the engine @Override - public BytesRef binaryValue() { + public IndexableFieldType fieldType() { throw tragicException; } }); diff --git a/server/src/test/java/org/opensearch/index/mapper/IgnoredFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/IgnoredFieldTypeTests.java index 4749c3280c89e..ef1530557d587 100644 --- a/server/src/test/java/org/opensearch/index/mapper/IgnoredFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/IgnoredFieldTypeTests.java @@ -33,11 +33,14 @@ package org.opensearch.index.mapper; import org.apache.lucene.index.Term; +import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.RegexpQuery; import org.apache.lucene.search.WildcardQuery; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.automaton.Operations; +import org.apache.lucene.util.automaton.RegExp; import org.opensearch.OpenSearchException; public class IgnoredFieldTypeTests extends FieldTypeTestCase { @@ -45,7 +48,7 @@ public class IgnoredFieldTypeTests extends FieldTypeTestCase { public void testPrefixQuery() { MappedFieldType ft = IgnoredFieldMapper.IgnoredFieldType.INSTANCE; - Query expected = new PrefixQuery(new Term("_ignored", new BytesRef("foo*"))); + Query expected = new PrefixQuery(new Term("_ignored", new BytesRef("foo*")), MultiTermQuery.CONSTANT_SCORE_REWRITE); assertEquals(expected, ft.prefixQuery("foo*", null, MOCK_QSC)); OpenSearchException ee = expectThrows(OpenSearchException.class, () -> ft.prefixQuery("foo*", null, MOCK_QSC_DISALLOW_EXPENSIVE)); @@ -59,7 +62,14 @@ public void testPrefixQuery() { public void testRegexpQuery() { MappedFieldType ft = IgnoredFieldMapper.IgnoredFieldType.INSTANCE; - Query expected = new RegexpQuery(new Term("_ignored", new BytesRef("foo?"))); + Query expected = new RegexpQuery( + new Term("_ignored", new BytesRef("foo?")), + RegExp.ALL, + 0, + RegexpQuery.DEFAULT_PROVIDER, + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, + MultiTermQuery.CONSTANT_SCORE_REWRITE + ); assertEquals(expected, ft.regexpQuery("foo?", 0, 0, 10, null, MOCK_QSC)); OpenSearchException ee = expectThrows( @@ -73,7 +83,7 @@ public void testWildcardQuery() { MappedFieldType ft = IgnoredFieldMapper.IgnoredFieldType.INSTANCE; Query expected = new WildcardQuery(new Term("_ignored", new BytesRef("foo*"))); - assertEquals(expected, ft.wildcardQuery("foo*", null, MOCK_QSC)); + assertEquals(expected, ft.wildcardQuery("foo*", MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, MOCK_QSC)); OpenSearchException ee = expectThrows( OpenSearchException.class, diff --git a/server/src/test/java/org/opensearch/index/mapper/KeywordFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/KeywordFieldTypeTests.java index ad529c685d6f3..626af63fc968d 100644 --- a/server/src/test/java/org/opensearch/index/mapper/KeywordFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/KeywordFieldTypeTests.java @@ -43,6 +43,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.FuzzyQuery; +import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.NormsFieldExistsQuery; import org.apache.lucene.search.RegexpQuery; import org.apache.lucene.search.TermInSetQuery; @@ -172,7 +173,10 @@ public void testRangeQuery() { public void testRegexpQuery() { MappedFieldType ft = new KeywordFieldType("field"); - assertEquals(new RegexpQuery(new Term("field", "foo.*")), ft.regexpQuery("foo.*", 0, 0, 10, null, MOCK_QSC)); + assertEquals( + new RegexpQuery(new Term("field", "foo.*")), + ft.regexpQuery("foo.*", 0, 0, 10, MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, MOCK_QSC) + ); MappedFieldType unsearchable = new KeywordFieldType("field", false, true, Collections.emptyMap()); IllegalArgumentException e = expectThrows( diff --git a/server/src/test/java/org/opensearch/index/mapper/RoutingFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/RoutingFieldTypeTests.java index 8d3a8ea43f264..7553a6c621f62 100644 --- a/server/src/test/java/org/opensearch/index/mapper/RoutingFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/RoutingFieldTypeTests.java @@ -32,6 +32,7 @@ package org.opensearch.index.mapper; import org.apache.lucene.index.Term; +import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.RegexpQuery; @@ -45,7 +46,7 @@ public void testPrefixQuery() { MappedFieldType ft = RoutingFieldMapper.RoutingFieldType.INSTANCE; Query expected = new PrefixQuery(new Term("_routing", new BytesRef("foo*"))); - assertEquals(expected, ft.prefixQuery("foo*", null, MOCK_QSC)); + assertEquals(expected, ft.prefixQuery("foo*", MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, MOCK_QSC)); OpenSearchException ee = expectThrows(OpenSearchException.class, () -> ft.prefixQuery("foo*", null, MOCK_QSC_DISALLOW_EXPENSIVE)); assertEquals( @@ -59,7 +60,7 @@ public void testRegexpQuery() { MappedFieldType ft = RoutingFieldMapper.RoutingFieldType.INSTANCE; Query expected = new RegexpQuery(new Term("_routing", new BytesRef("foo?"))); - assertEquals(expected, ft.regexpQuery("foo?", 0, 0, 10, null, MOCK_QSC)); + assertEquals(expected, ft.regexpQuery("foo?", 0, 0, 10, MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, MOCK_QSC)); OpenSearchException ee = expectThrows( OpenSearchException.class, @@ -72,7 +73,7 @@ public void testWildcardQuery() { MappedFieldType ft = RoutingFieldMapper.RoutingFieldType.INSTANCE; Query expected = new WildcardQuery(new Term("_routing", new BytesRef("foo*"))); - assertEquals(expected, ft.wildcardQuery("foo*", null, MOCK_QSC)); + assertEquals(expected, ft.wildcardQuery("foo*", MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, MOCK_QSC)); OpenSearchException ee = expectThrows( OpenSearchException.class, diff --git a/server/src/test/java/org/opensearch/index/mapper/TextFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/TextFieldTypeTests.java index 206be8c8352ef..b8d470e6bdf3e 100644 --- a/server/src/test/java/org/opensearch/index/mapper/TextFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/TextFieldTypeTests.java @@ -61,6 +61,7 @@ import java.util.List; import static org.apache.lucene.search.MultiTermQuery.CONSTANT_SCORE_REWRITE; +import static org.apache.lucene.search.MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE; import static org.hamcrest.Matchers.equalTo; public class TextFieldTypeTests extends FieldTypeTestCase { @@ -120,7 +121,10 @@ public void testRangeQuery() { public void testRegexpQuery() { MappedFieldType ft = createFieldType(); - assertEquals(new RegexpQuery(new Term("field", "foo.*")), ft.regexpQuery("foo.*", 0, 0, 10, null, MOCK_QSC)); + assertEquals( + new RegexpQuery(new Term("field", "foo.*")), + ft.regexpQuery("foo.*", 0, 0, 10, CONSTANT_SCORE_BLENDED_REWRITE, MOCK_QSC) + ); MappedFieldType unsearchable = new TextFieldType("field", false, false, Collections.emptyMap()); IllegalArgumentException e = expectThrows( @@ -165,7 +169,7 @@ public void testIndexPrefixes() { assertEquals(new ConstantScoreQuery(new TermQuery(new Term("field._index_prefix", "goin"))), q); q = ft.prefixQuery("internationalisatio", CONSTANT_SCORE_REWRITE, false, MOCK_QSC); - assertEquals(new PrefixQuery(new Term("field", "internationalisatio")), q); + assertEquals(new PrefixQuery(new Term("field", "internationalisatio"), CONSTANT_SCORE_REWRITE), q); q = ft.prefixQuery("Internationalisatio", CONSTANT_SCORE_REWRITE, true, MOCK_QSC); assertEquals(AutomatonQueries.caseInsensitivePrefixQuery(new Term("field", "Internationalisatio")), q); @@ -184,9 +188,16 @@ public void testIndexPrefixes() { Automaton automaton = Operations.concatenate(Arrays.asList(Automata.makeChar('g'), Automata.makeAnyChar())); Query expected = new ConstantScoreQuery( - new BooleanQuery.Builder().add(new AutomatonQuery(new Term("field._index_prefix", "g*"), automaton), BooleanClause.Occur.SHOULD) - .add(new TermQuery(new Term("field", "g")), BooleanClause.Occur.SHOULD) - .build() + new BooleanQuery.Builder().add( + new AutomatonQuery( + new Term("field._index_prefix", "g*"), + automaton, + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, + false, + CONSTANT_SCORE_REWRITE + ), + BooleanClause.Occur.SHOULD + ).add(new TermQuery(new Term("field", "g")), BooleanClause.Occur.SHOULD).build() ); assertThat(q, equalTo(expected)); @@ -195,9 +206,16 @@ public void testIndexPrefixes() { automaton = Operations.concatenate(Arrays.asList(Automata.makeChar('g'), Automata.makeAnyChar())); expected = new ConstantScoreQuery( - new BooleanQuery.Builder().add(new AutomatonQuery(new Term("field._index_prefix", "g*"), automaton), BooleanClause.Occur.SHOULD) - .add(new TermQuery(new Term("field", "g")), BooleanClause.Occur.SHOULD) - .build() + new BooleanQuery.Builder().add( + new AutomatonQuery( + new Term("field._index_prefix", "g*"), + automaton, + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, + false, + CONSTANT_SCORE_REWRITE + ), + BooleanClause.Occur.SHOULD + ).add(new TermQuery(new Term("field", "g")), BooleanClause.Occur.SHOULD).build() ); assertThat(q, equalTo(expected)); diff --git a/server/src/test/java/org/opensearch/index/query/DisMaxQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/DisMaxQueryBuilderTests.java index bc8cd6c57b975..8d589bd76f2bb 100644 --- a/server/src/test/java/org/opensearch/index/query/DisMaxQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/DisMaxQueryBuilderTests.java @@ -35,6 +35,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.DisjunctionMaxQuery; +import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; import org.opensearch.test.AbstractQueryTestCase; @@ -103,7 +104,10 @@ public void testToQueryInnerPrefixQuery() throws Exception { + " }\n" + "}"; Query query = parseQuery(queryAsString).toQuery(createShardContext()); - Query expected = new DisjunctionMaxQuery(List.of(new BoostQuery(new PrefixQuery(new Term(TEXT_FIELD_NAME, "sh")), 1.2f)), 0); + Query expected = new DisjunctionMaxQuery( + List.of(new BoostQuery(new PrefixQuery(new Term(TEXT_FIELD_NAME, "sh"), MultiTermQuery.CONSTANT_SCORE_REWRITE), 1.2f)), + 0 + ); assertEquals(expected, query); } diff --git a/server/src/test/java/org/opensearch/index/query/MatchBoolPrefixQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/MatchBoolPrefixQueryBuilderTests.java index 0e3fd0efe7804..a23b4fd75e8a0 100644 --- a/server/src/test/java/org/opensearch/index/query/MatchBoolPrefixQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/MatchBoolPrefixQueryBuilderTests.java @@ -37,6 +37,7 @@ import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.FuzzyQuery; +import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.SynonymQuery; @@ -268,7 +269,7 @@ public void testAnalysis() throws Exception { asList( new TermQuery(new Term(TEXT_FIELD_NAME, "foo")), new TermQuery(new Term(TEXT_FIELD_NAME, "bar")), - new PrefixQuery(new Term(TEXT_FIELD_NAME, "baz")) + new PrefixQuery(new Term(TEXT_FIELD_NAME, "baz"), MultiTermQuery.CONSTANT_SCORE_REWRITE) ) ); } @@ -285,7 +286,7 @@ public void testAnalysisSynonym() throws Exception { new SynonymQuery.Builder(TEXT_FIELD_NAME).addTerm(new Term(TEXT_FIELD_NAME, "dogs")) .addTerm(new Term(TEXT_FIELD_NAME, "dog")) .build(), - new PrefixQuery(new Term(TEXT_FIELD_NAME, "red")) + new PrefixQuery(new Term(TEXT_FIELD_NAME, "red"), MultiTermQuery.CONSTANT_SCORE_REWRITE) ) ); } @@ -293,7 +294,7 @@ public void testAnalysisSynonym() throws Exception { public void testAnalysisSingleTerm() throws Exception { final MatchBoolPrefixQueryBuilder builder = new MatchBoolPrefixQueryBuilder(TEXT_FIELD_NAME, "foo"); final Query query = builder.toQuery(createShardContext()); - assertThat(query, equalTo(new PrefixQuery(new Term(TEXT_FIELD_NAME, "foo")))); + assertThat(query, equalTo(new PrefixQuery(new Term(TEXT_FIELD_NAME, "foo"), MultiTermQuery.CONSTANT_SCORE_REWRITE))); } private static void assertBooleanQuery(Query actual, List expectedClauseQueries) { diff --git a/server/src/test/java/org/opensearch/index/query/QueryStringQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/QueryStringQueryBuilderTests.java index 40fac046dfbb9..03ddb625c78a8 100644 --- a/server/src/test/java/org/opensearch/index/query/QueryStringQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/QueryStringQueryBuilderTests.java @@ -567,7 +567,12 @@ public void testToQueryWildcardQuery() throws Exception { .add( new BooleanQuery.Builder().add(new BooleanClause(new TermQuery(new Term(TEXT_FIELD_NAME, "foo")), defaultOp)) .add(new BooleanClause(new TermQuery(new Term(TEXT_FIELD_NAME, "bar")), defaultOp)) - .add(new BooleanClause(new PrefixQuery(new Term(TEXT_FIELD_NAME, "foobar")), defaultOp)) + .add( + new BooleanClause( + new PrefixQuery(new Term(TEXT_FIELD_NAME, "foobar"), MultiTermQuery.CONSTANT_SCORE_REWRITE), + defaultOp + ) + ) .build(), defaultOp ) @@ -579,6 +584,8 @@ public void testToQueryWildcardQuery() throws Exception { public void testToQueryWildcardWithIndexedPrefixes() throws Exception { QueryStringQueryParser queryParser = new QueryStringQueryParser(createShardContext(), "prefix_field"); + queryParser.setMultiTermRewriteMethod(MultiTermQuery.CONSTANT_SCORE_REWRITE); + Query query = queryParser.parse("foo*"); Query expectedQuery = new ConstantScoreQuery(new TermQuery(new Term("prefix_field._index_prefix", "foo"))); assertThat(query, equalTo(expectedQuery)); @@ -586,9 +593,16 @@ public void testToQueryWildcardWithIndexedPrefixes() throws Exception { query = queryParser.parse("g*"); Automaton a = Operations.concatenate(Arrays.asList(Automata.makeChar('g'), Automata.makeAnyChar())); expectedQuery = new ConstantScoreQuery( - new BooleanQuery.Builder().add(new AutomatonQuery(new Term("prefix_field._index_prefix", "g*"), a), Occur.SHOULD) - .add(new TermQuery(new Term("prefix_field", "g")), Occur.SHOULD) - .build() + new BooleanQuery.Builder().add( + new AutomatonQuery( + new Term("prefix_field._index_prefix", "g*"), + a, + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, + false, + MultiTermQuery.CONSTANT_SCORE_REWRITE + ), + Occur.SHOULD + ).add(new TermQuery(new Term("prefix_field", "g")), Occur.SHOULD).build() ); assertThat(query, equalTo(expectedQuery)); } @@ -598,7 +612,7 @@ public void testToQueryWildcardQueryWithSynonyms() throws Exception { BooleanClause.Occur defaultOp = op.toBooleanClauseOccur(); QueryStringQueryParser queryParser = new QueryStringQueryParser(createShardContext(), TEXT_FIELD_NAME); queryParser.setAnalyzeWildcard(true); - queryParser.setMultiTermRewriteMethod(MultiTermQuery.CONSTANT_SCORE_REWRITE); + queryParser.setMultiTermRewriteMethod(MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE); queryParser.setDefaultOperator(op.toQueryParserOperator()); queryParser.setForceAnalyzer(new MockRepeatAnalyzer()); Query query = queryParser.parse("first foo-bar-foobar* last"); @@ -1157,15 +1171,29 @@ public void testFromJson() throws IOException { public void testExpandedTerms() throws Exception { // Prefix Query query = new QueryStringQueryBuilder("aBc*").field(TEXT_FIELD_NAME).analyzer("whitespace").toQuery(createShardContext()); - assertEquals(new PrefixQuery(new Term(TEXT_FIELD_NAME, "aBc")), query); + assertEquals(new PrefixQuery(new Term(TEXT_FIELD_NAME, "aBc"), MultiTermQuery.CONSTANT_SCORE_REWRITE), query); query = new QueryStringQueryBuilder("aBc*").field(TEXT_FIELD_NAME).analyzer("standard").toQuery(createShardContext()); - assertEquals(new PrefixQuery(new Term(TEXT_FIELD_NAME, "abc")), query); + assertEquals(new PrefixQuery(new Term(TEXT_FIELD_NAME, "abc"), MultiTermQuery.CONSTANT_SCORE_REWRITE), query); // Wildcard query = new QueryStringQueryBuilder("aBc*D").field(TEXT_FIELD_NAME).analyzer("whitespace").toQuery(createShardContext()); - assertEquals(new WildcardQuery(new Term(TEXT_FIELD_NAME, "aBc*D")), query); + assertEquals( + new WildcardQuery( + new Term(TEXT_FIELD_NAME, "aBc*D"), + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, + MultiTermQuery.CONSTANT_SCORE_REWRITE + ), + query + ); query = new QueryStringQueryBuilder("aBc*D").field(TEXT_FIELD_NAME).analyzer("standard").toQuery(createShardContext()); - assertEquals(new WildcardQuery(new Term(TEXT_FIELD_NAME, "abc*d")), query); + assertEquals( + new WildcardQuery( + new Term(TEXT_FIELD_NAME, "abc*d"), + Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, + MultiTermQuery.CONSTANT_SCORE_REWRITE + ), + query + ); // Fuzzy query = new QueryStringQueryBuilder("aBc~1").field(TEXT_FIELD_NAME).analyzer("whitespace").toQuery(createShardContext()); @@ -1407,7 +1435,10 @@ public void testEnablePositionIncrement() throws Exception { public void testWithPrefixStopWords() throws Exception { Query query = new QueryStringQueryBuilder("the* quick fox").field(TEXT_FIELD_NAME).analyzer("stop").toQuery(createShardContext()); - BooleanQuery expected = new BooleanQuery.Builder().add(new PrefixQuery(new Term(TEXT_FIELD_NAME, "the")), Occur.SHOULD) + BooleanQuery expected = new BooleanQuery.Builder().add( + new PrefixQuery(new Term(TEXT_FIELD_NAME, "the"), MultiTermQuery.CONSTANT_SCORE_REWRITE), + Occur.SHOULD + ) .add(new TermQuery(new Term(TEXT_FIELD_NAME, "quick")), Occur.SHOULD) .add(new TermQuery(new Term(TEXT_FIELD_NAME, "fox")), Occur.SHOULD) .build(); @@ -1488,7 +1519,7 @@ public void testAnalyzedPrefix() throws Exception { .analyzer("standard") .analyzeWildcard(true) .toQuery(createShardContext()); - Query expected = new PrefixQuery(new Term(TEXT_FIELD_NAME, "quick")); + Query expected = new PrefixQuery(new Term(TEXT_FIELD_NAME, "quick"), MultiTermQuery.CONSTANT_SCORE_REWRITE); assertEquals(expected, query); } diff --git a/server/src/test/java/org/opensearch/index/query/SimpleQueryStringBuilderTests.java b/server/src/test/java/org/opensearch/index/query/SimpleQueryStringBuilderTests.java index 1101da8de70fe..7d31240b1a665 100644 --- a/server/src/test/java/org/opensearch/index/query/SimpleQueryStringBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/SimpleQueryStringBuilderTests.java @@ -45,6 +45,7 @@ import org.apache.lucene.search.DisjunctionMaxQuery; import org.apache.lucene.search.FuzzyQuery; import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; @@ -451,9 +452,9 @@ public void testMinimumShouldMatch() throws IOException { public void testExpandedTerms() throws Exception { // Prefix Query query = new SimpleQueryStringBuilder("aBc*").field(TEXT_FIELD_NAME).analyzer("whitespace").toQuery(createShardContext()); - assertEquals(new PrefixQuery(new Term(TEXT_FIELD_NAME, "aBc")), query); + assertEquals(new PrefixQuery(new Term(TEXT_FIELD_NAME, "aBc"), MultiTermQuery.CONSTANT_SCORE_REWRITE), query); query = new SimpleQueryStringBuilder("aBc*").field(TEXT_FIELD_NAME).analyzer("standard").toQuery(createShardContext()); - assertEquals(new PrefixQuery(new Term(TEXT_FIELD_NAME, "abc")), query); + assertEquals(new PrefixQuery(new Term(TEXT_FIELD_NAME, "abc"), MultiTermQuery.CONSTANT_SCORE_REWRITE), query); // Fuzzy query = new SimpleQueryStringBuilder("aBc~1").field(TEXT_FIELD_NAME).analyzer("whitespace").toQuery(createShardContext()); @@ -712,8 +713,12 @@ public void testLenientToPrefixQuery() throws Exception { .lenient(true) .toQuery(createShardContext()); List expectedQueries = new ArrayList<>(); - expectedQueries.add(new MatchNoDocsQuery("")); - expectedQueries.add(new PrefixQuery(new Term(TEXT_FIELD_NAME, "t"))); + expectedQueries.add( + new MatchNoDocsQuery( + "failed query, caused by Can only use prefix queries on keyword and text fields - not on [mapped_date] which is of type [date]" + ) + ); + expectedQueries.add(new PrefixQuery(new Term(TEXT_FIELD_NAME, "t"), MultiTermQuery.CONSTANT_SCORE_REWRITE)); DisjunctionMaxQuery expected = new DisjunctionMaxQuery(expectedQueries, 1.0f); assertEquals(expected, query); } @@ -766,7 +771,7 @@ public void testWithStopWords() throws Exception { public void testWithPrefixStopWords() throws Exception { Query query = new SimpleQueryStringBuilder("the* quick fox").field(TEXT_FIELD_NAME).analyzer("stop").toQuery(createShardContext()); BooleanQuery expected = new BooleanQuery.Builder().add( - new PrefixQuery(new Term(TEXT_FIELD_NAME, "the")), + new PrefixQuery(new Term(TEXT_FIELD_NAME, "the"), MultiTermQuery.CONSTANT_SCORE_REWRITE), BooleanClause.Occur.SHOULD ) .add(new TermQuery(new Term(TEXT_FIELD_NAME, "quick")), BooleanClause.Occur.SHOULD)