Merge branch 'main' into main-opentelemetry

Signed-off-by: suraj kumar <[email protected]>

CHANGELOG.md

@@ -97,13 +97,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `netty` from 4.1.91.Final to 4.1.93.Final ([#7901](https://github.com/opensearch-project/OpenSearch/pull/7901))
 - Bump `com.amazonaws` 1.12.270 to `software.amazon.awssdk` 2.20.55 ([7372](https://github.com/opensearch-project/OpenSearch/pull/7372/))
 - Add `org.reactivestreams` 1.0.4 ([7372](https://github.com/opensearch-project/OpenSearch/pull/7372/))
-- Bump `com.networknt:json-schema-validator` from 1.0.81 to 1.0.83 ([7968](https://github.com/opensearch-project/OpenSearch/pull/7968))
+- Bump `com.networknt:json-schema-validator` from 1.0.81 to 1.0.85 ([7968], #8255)
 - Bump `com.netflix.nebula:gradle-extra-configurations-plugin` from 9.0.0 to 10.0.0 in /buildSrc ([#7068](https://github.com/opensearch-project/OpenSearch/pull/7068))
 - Bump `com.google.guava:guava` from 32.0.0-jre to 32.0.1-jre (#8009)
 - Bump `commons-io:commons-io` from 2.12.0 to 2.13.0 (#8014, #8013, #8010)
 - Bump `com.diffplug.spotless` from 6.18.0 to 6.19.0 (#8007)
 - Bump `'com.azure:azure-storage-blob` to 12.22.2 from 12.21.1 ([#8043](https://github.com/opensearch-project/OpenSearch/pull/8043))
-- Bump `org.jruby.joni:joni` from 2.1.48 to 2.2.1 (#8015)
+- Bump `org.jruby.joni:joni` from 2.1.48 to 2.2.1 (#8015, #8254)
 - Bump `com.google.guava:guava` from 32.0.0-jre to 32.0.1-jre ([#8011](https://github.com/opensearch-project/OpenSearch/pull/8011), [#8012](https://github.com/opensearch-project/OpenSearch/pull/8012), [#8107](https://github.com/opensearch-project/OpenSearch/pull/8107))
 - Bump `io.projectreactor:reactor-core` from 3.4.18 to 3.5.6 in /plugins/repository-azure ([#8016](https://github.com/opensearch-project/OpenSearch/pull/8016))
 - Bump `spock-core` from 2.1-groovy-3.0 to 2.3-groovy-3.0 ([#8122](https://github.com/opensearch-project/OpenSearch/pull/8122))
@@ -112,6 +112,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `commons-io:commons-io` from 2.12.0 to 2.13.0 in /plugins/discovery-azure-classic ([#8140](https://github.com/opensearch-project/OpenSearch/pull/8140))
 - Bump `mockito` from 5.2.0 to 5.4.0 ([#8181](https://github.com/opensearch-project/OpenSearch/pull/8181))
 - Bump `netty` from 4.1.93.Final to 4.1.94.Final ([#8191](https://github.com/opensearch-project/OpenSearch/pull/8191))
+- Bump `org.apache.hadoop:hadoop-minicluster` from 3.3.5 to 3.3.6 (#8257)
+- Bump `io.projectreactor.netty:reactor-netty-http` from 1.1.7 to 1.1.8 (#8256)
+- [Upgrade] Lucene 9.7.0 release (#8272)
+- Bump `org.jboss.resteasy:resteasy-jackson2-provider` from 3.0.26.Final to 6.2.4.Final in /qa/wildfly ([#8209](https://github.com/opensearch-project/OpenSearch/pull/8209))
 
 ### Changed
 - Replace jboss-annotations-api_1.2_spec with jakarta.annotation-api ([#7836](https://github.com/opensearch-project/OpenSearch/pull/7836))

buildSrc/build.gradle

@@ -118,9 +118,9 @@ dependencies {
   api 'com.avast.gradle:gradle-docker-compose-plugin:0.16.12'
   api "org.yaml:snakeyaml:${props.getProperty('snakeyaml')}"
   api 'org.apache.maven:maven-model:3.9.2'
-  api 'com.networknt:json-schema-validator:1.0.84'
+  api 'com.networknt:json-schema-validator:1.0.85'
   api 'org.jruby.jcodings:jcodings:1.0.58'
-  api 'org.jruby.joni:joni:2.1.48'
+  api 'org.jruby.joni:joni:2.2.1'
   api "com.fasterxml.jackson.core:jackson-databind:${props.getProperty('jackson_databind')}"
   api "org.ajoberstar.grgit:grgit-core:5.2.0"
 

buildSrc/version.properties

@@ -1,5 +1,5 @@
 opensearch        = 3.0.0
-lucene            = 9.7.0-snapshot-204acc3
+lucene            = 9.7.0
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 20.0.1+9
@@ -65,6 +65,7 @@ zstd              = 1.5.5-3
 
 jzlib             = 1.1.3
 
+resteasy          = 6.2.4.Final
+
 # opentelemetry dependencies
 opentelemetry    = 1.26.0
-

libs/core/licenses/lucene-core-9.7.0.jar.sha1

@@ -0,0 +1 @@
+ad391210ffd806931334be9670a35af00c56f959

modules/lang-expression/licenses/lucene-expressions-9.7.0.jar.sha1

@@ -0,0 +1 @@
+297e1cfade4ef71466cc9d4f361d81807c8dc4c8

plugins/analysis-icu/licenses/lucene-analysis-icu-9.7.0.jar.sha1

@@ -0,0 +1 @@
+94293b169fb8572f440a5a4a523320ecf9778ffe

plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.7.0.jar.sha1

@@ -0,0 +1 @@
+2df800a38b64867b8dcd61fc2cd986114e4a80cb

plugins/analysis-nori/licenses/lucene-analysis-nori-9.7.0.jar.sha1

@@ -0,0 +1 @@
+a01e8153f34d72e8c8c0180c1dea5b10f677dd3a

plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.7.0.jar.sha1

@@ -0,0 +1 @@
+b7d47d54683b0b1e09b271c32d1b7d3eb1990f49

plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.7.0.jar.sha1

@@ -0,0 +1 @@
+5e68b9816e6cff8ee15f5b350cf2ffa54f9828b7

plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.7.0.jar.sha1

@@ -0,0 +1 @@
+d23b1f05b471e05d0d6068b3ece7c8c65672eae7

plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.7.0.jar.sha1

@@ -0,0 +1 @@
+dfb4313f3c68d337310522840d7144c1605d084a

plugins/repository-azure/build.gradle

@@ -60,7 +60,7 @@ dependencies {
   api 'io.projectreactor:reactor-core:3.5.6'
   api 'io.projectreactor.netty:reactor-netty:1.1.7'
   api 'io.projectreactor.netty:reactor-netty-core:1.1.7'
-  api 'io.projectreactor.netty:reactor-netty-http:1.1.7'
+  api 'io.projectreactor.netty:reactor-netty-http:1.1.8'
   api "org.slf4j:slf4j-api:${versions.slf4j}"
   api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
   api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"

plugins/repository-azure/licenses/reactor-netty-http-1.1.8.jar.sha1

@@ -0,0 +1 @@
+696ea25658295e49906c6aad13fa70acbdeb2359

qa/mixed-cluster/build.gradle

@@ -55,6 +55,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) {
   }
 
   String baseName = "v${bwcVersion}"
+  String bwcVersionStr = "${bwcVersion}"
 
   /* This project runs the core REST tests against a 4 node cluster where two of
      the nodes has a different minor.  */
@@ -86,6 +87,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) {
       nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
       nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
     }
+    systemProperty 'tests.upgrade_from_version', bwcVersionStr
     systemProperty 'tests.path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
     onlyIf { project.bwc_tests_enabled }
   }

qa/mixed-cluster/src/test/java/org/opensearch/backwards/IndexingIT.java

@@ -66,6 +66,9 @@
 
 public class IndexingIT extends OpenSearchRestTestCase {
 
+    protected static final Version UPGRADE_FROM_VERSION = Version.fromString(System.getProperty("tests.upgrade_from_version"));
+
+
     private int indexDocs(String index, final int idStart, final int numDocs) throws IOException {
         for (int i = 0; i < numDocs; i++) {
             final int id = idStart + i;
@@ -114,12 +117,16 @@ private void printClusterRouting() throws IOException, ParseException {
      * @throws Exception
      */
     public void testIndexingWithPrimaryOnBwcNodes() throws Exception {
+        if (UPGRADE_FROM_VERSION.before(Version.V_2_4_0)) {
+            logger.info("--> Skip test for version {} where segment replication feature is not available", UPGRADE_FROM_VERSION);
+            return;
+        }
         Nodes nodes = buildNodeAndVersions();
         assumeFalse("new nodes is empty", nodes.getNewNodes().isEmpty());
         logger.info("cluster discovered:\n {}", nodes.toString());
         final List<String> bwcNamesList = nodes.getBWCNodes().stream().map(Node::getNodeName).collect(Collectors.toList());
         final String bwcNames = bwcNamesList.stream().collect(Collectors.joining(","));
-        // Exclude bwc nodes from allocation so that primaries gets allocated on current version
+        // Update allocation settings so that primaries gets allocated only on nodes running on older version
         Settings.Builder settings = Settings.builder()
             .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
             .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
@@ -133,7 +140,7 @@ public void testIndexingWithPrimaryOnBwcNodes() throws Exception {
         try (RestClient nodeClient = buildClient(restClientSettings(),
             nodes.getNewNodes().stream().map(Node::getPublishAddress).toArray(HttpHost[]::new))) {
 
-            logger.info("allowing replica shards assignment on bwc nodes");
+            logger.info("Remove allocation include settings so that shards can be allocated on current version nodes");
             updateIndexSettings(index, Settings.builder().putNull("index.routing.allocation.include._name"));
             // Add replicas so that it can be assigned on higher OS version nodes.
             updateIndexSettings(index, Settings.builder().put("index.number_of_replicas", 2));
@@ -154,13 +161,17 @@ public void testIndexingWithPrimaryOnBwcNodes() throws Exception {
 
 
     /**
-     * This test creates a cluster with primary on older version but due to {@link org.opensearch.cluster.routing.allocation.decider.NodeVersionAllocationDecider};
+     * This test creates a cluster with primary on higher version but due to {@link org.opensearch.cluster.routing.allocation.decider.NodeVersionAllocationDecider};
      * replica shard allocation on lower OpenSearch version is prevented. Thus, this test though cover the use case where
      * primary shard containing nodes are running on higher OS version while replicas are unassigned.
      *
      * @throws Exception
      */
     public void testIndexingWithReplicaOnBwcNodes() throws Exception {
+        if (UPGRADE_FROM_VERSION.before(Version.V_2_4_0)) {
+            logger.info("--> Skip test for version {} where segment replication feature is not available", UPGRADE_FROM_VERSION);
+            return;
+        }
         Nodes nodes = buildNodeAndVersions();
         assumeFalse("new nodes is empty", nodes.getNewNodes().isEmpty());
         logger.info("cluster discovered:\n {}", nodes.toString());

qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java

@@ -38,13 +38,17 @@
 import org.opensearch.client.Response;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.Booleans;
+import org.opensearch.common.io.Streams;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.index.codec.CodecService;
+import org.opensearch.index.engine.EngineConfig;
 import org.opensearch.indices.replication.common.ReplicationType;
 import org.opensearch.test.rest.yaml.ObjectPath;
 
 import java.io.IOException;
 import java.net.URISyntaxException;
 import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -90,6 +94,7 @@ private void waitForSearchableDocs(String index, int shardCount) throws Exceptio
         waitForClusterHealthWithNoShardMigration(index, "green");
         logger.info("--> _cat/shards before search \n{}", EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/shards?v")).getEntity()));
 
+        verifySegmentStats(index);
         Request request = new Request("GET", index + "/_stats");
         request.addParameter("level", "shards");
         Response response = client().performRequest(request);
@@ -109,14 +114,12 @@ private void waitForSearchableDocs(String index, int shardCount) throws Exceptio
         logger.info("--> replicaShardToNodeIDMap {}", replicaShardToNodeIDMap);
 
         for (int shardNumber = 0; shardNumber < shardCount; shardNumber++) {
-            logger.info("--> Verify doc count for shard number {}", shardNumber);
             Request searchTestIndexRequest = new Request("POST", "/" + index + "/_search");
             searchTestIndexRequest.addParameter(TOTAL_HITS_AS_INT_PARAM, "true");
             searchTestIndexRequest.addParameter("filter_path", "hits.total");
             searchTestIndexRequest.addParameter("preference", "_shards:" + shardNumber + "|_only_nodes:" + primaryShardToNodeIDMap.get(shardNumber));
             Response searchTestIndexResponse = client().performRequest(searchTestIndexRequest);
             final int primaryHits = ObjectPath.createFromResponse(searchTestIndexResponse).evaluate("hits.total");
-            logger.info("--> primaryHits {}", primaryHits);
             final int shardNum = shardNumber;
             // Verify replica shard doc count only when available.
             if (replicaShardToNodeIDMap.get(shardNum) != null) {
@@ -127,8 +130,7 @@ private void waitForSearchableDocs(String index, int shardCount) throws Exceptio
                     replicaRequest.addParameter("preference", "_shards:" + shardNum + "|_only_nodes:" + replicaShardToNodeIDMap.get(shardNum));
                     Response replicaResponse = client().performRequest(replicaRequest);
                     int replicaHits = ObjectPath.createFromResponse(replicaResponse).evaluate("hits.total");
-                    logger.info("--> ReplicaHits {}", replicaHits);
-                    assertEquals(primaryHits, replicaHits);
+                    assertEquals("Doc count mismatch for shard " + shardNum + ". Primary hits " + primaryHits + " Replica hits " + replicaHits, primaryHits, replicaHits);
                 }, 1, TimeUnit.MINUTES);
             }
         }
@@ -145,6 +147,18 @@ private void waitForClusterHealthWithNoShardMigration(String indexName, String s
         client().performRequest(waitForStatus);
     }
 
+    private void verifySegmentStats(String indexName) throws Exception {
+        assertBusy(() -> {
+            Request segrepStatsRequest = new Request("GET", "/_cat/segment_replication/" + indexName);
+            segrepStatsRequest.addParameter("h", "shardId,target_node,checkpoints_behind");
+            Response segrepStatsResponse = client().performRequest(segrepStatsRequest);
+            for (String statLine : Streams.readAllLines(segrepStatsResponse.getEntity().getContent())) {
+                String[] elements = statLine.split(" +");
+                assertEquals("Replica shard " + elements[0] + "not upto date with primary ", 0, Integer.parseInt(elements[2]));
+            }
+        });
+    }
+
     public void testIndexing() throws IOException, ParseException {
         switch (CLUSTER_TYPE) {
         case OLD:
@@ -239,9 +253,13 @@ public void testIndexing() throws IOException, ParseException {
      * @throws Exception
      */
     public void testIndexingWithSegRep() throws Exception {
+        if (UPGRADE_FROM_VERSION.before(Version.V_2_4_0)) {
+            logger.info("--> Skip test for version {} where segment replication feature is not available", UPGRADE_FROM_VERSION);
+            return;
+        }
         final String indexName = "test-index-segrep";
         final int shardCount = 3;
-        final int replicaCount = 1;
+        final int replicaCount = 2;
         logger.info("--> Case {}", CLUSTER_TYPE);
         printClusterNodes();
         logger.info("--> _cat/shards before test execution \n{}", EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/shards?v")).getEntity()));
@@ -251,6 +269,10 @@ public void testIndexingWithSegRep() throws Exception {
                     .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), shardCount)
                     .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), replicaCount)
                     .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
+                    .put(
+                        EngineConfig.INDEX_CODEC_SETTING.getKey(),
+                        randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC, CodecService.LUCENE_DEFAULT_CODEC)
+                    )
                     .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms");
                 createIndex(indexName, settings.build());
                 waitForClusterHealthWithNoShardMigration(indexName, "green");

qa/wildfly/build.gradle

@@ -40,25 +40,32 @@ apply plugin: 'opensearch.internal-distribution-download'
 testFixtures.useFixture()
 
 dependencies {
-  providedCompile 'javax.enterprise:cdi-api:2.0'
-  providedCompile "jakarta.annotation:jakarta.annotation-api:${versions.jakarta_annotation}"
-  providedCompile 'jakarta.ws.rs:jakarta.ws.rs-api:2.1.3'
-  api('org.jboss.resteasy:resteasy-jackson2-provider:3.0.26.Final') {
-    exclude module: 'jackson-annotations'
-    exclude module: 'jackson-core'
-    exclude module: 'jackson-databind'
-    exclude module: 'jackson-jaxrs-json-provider'
+  providedCompile('jakarta.enterprise:jakarta.enterprise.cdi-api:4.0.1') {
+    exclude module: 'jakarta.annotation-api'
+  }
+  providedCompile 'jakarta.ws.rs:jakarta.ws.rs-api:3.1.0'
+  providedCompile "org.jboss.resteasy:resteasy-core:${versions.resteasy}"
+  providedCompile "org.jboss.resteasy:resteasy-core-spi:${versions.resteasy}"
+  api("org.jboss.resteasy:resteasy-jackson2-provider:${versions.resteasy}") {
+    exclude module: 'jakarta.activation-api'
+    exclude group: 'com.fasterxml.jackson'
+    exclude group: 'com.fasterxml.jackson.core'
+    exclude group: 'com.fasterxml.jackson.dataformat'
+    exclude group: 'com.fasterxml.jackson.module'
   }
   api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
-  api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
-  api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
-  api "com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:${versions.jackson}"
-  api "com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:${versions.jackson}"
-  api "com.fasterxml.jackson.module:jackson-module-jaxb-annotations:${versions.jackson}"
+  api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
+  api "com.fasterxml.jackson.jakarta.rs:jackson-jakarta-rs-base:${versions.jackson}"
+  api "com.fasterxml.jackson.jakarta.rs:jackson-jakarta-rs-json-provider:${versions.jackson}"
+  api "com.github.fge:json-patch:1.9"
   api "org.apache.logging.log4j:log4j-api:${versions.log4j}"
   api "org.apache.logging.log4j:log4j-core:${versions.log4j}"
-  api project(path: ':client:rest-high-level')
-  testImplementation project(':test:framework')
+  api(project(path: ':client:rest-high-level')) {
+    exclude module: 'jakarta.annotation-api'
+  }
+  testImplementation(project(':test:framework')) {
+    exclude module: 'jakarta.annotation-api'
+  }
 }
 
 war {

qa/wildfly/docker-compose.yml

@@ -2,7 +2,7 @@ version: '3.7'
 services:
 
   wildfly:
-    image: jboss/wildfly:18.0.1.Final
+    image: quay.io/wildfly/wildfly:28.0.1.Final-jdk11
     environment:
       JAVA_OPTS: -Dopensearch.uri=opensearch:9200 -Djboss.http.port=8080 -Djava.net.preferIPv4Stack=true
     volumes:

qa/wildfly/src/main/java/org/opensearch/wildfly/model/Employee.java

@@ -34,12 +34,8 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 
-import javax.ws.rs.Consumes;
-import javax.ws.rs.core.MediaType;
-
 import java.util.List;
 
-@Consumes(MediaType.APPLICATION_JSON)
 public class Employee {
 
     @JsonProperty(value = "first_name")

qa/wildfly/src/main/java/org/opensearch/wildfly/transport/RestHighLevelClientActivator.java

@@ -32,18 +32,10 @@
 
 package org.opensearch.wildfly.transport;
 
-import javax.ws.rs.ApplicationPath;
-import javax.ws.rs.core.Application;
-
-import java.util.Collections;
-import java.util.Set;
+import jakarta.ws.rs.ApplicationPath;
+import jakarta.ws.rs.core.Application;
 
 @ApplicationPath("/transport")
 public class RestHighLevelClientActivator extends Application {
 
-    @Override
-    public Set<Class<?>> getClasses() {
-        return Collections.singleton(RestHighLevelClientEmployeeResource.class);
-    }
-
 }