Merge branch 'seg-rep/new-checkpoint' of https://github.com/Rishikesh…

…1159/OpenSearch into seg-rep/new-checkpoint

.ci/bwcVersions

@@ -40,6 +40,7 @@ BWC_VERSION:
   - "1.3.1"
   - "1.3.2"
   - "1.3.3"
+  - "1.3.4"
   - "2.0.0"
   - "2.0.1"
   - "2.1.0"

buildSrc/version.properties

@@ -1,5 +1,5 @@
 opensearch        = 3.0.0
-lucene            = 9.3.0-snapshot-823df23
+lucene            = 9.3.0-snapshot-b7231bb
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 17.0.3+7

modules/lang-expression/licenses/lucene-expressions-9.3.0-snapshot-b7231bb.jar.sha1

@@ -0,0 +1 @@
+57ae445a0050ad492ef494b692b486dfe718b564

plugins/analysis-icu/licenses/lucene-analysis-icu-9.3.0-snapshot-b7231bb.jar.sha1

@@ -0,0 +1 @@
+b10e5bdae6df879b770060e0006bbc1c780c886d

plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.3.0-snapshot-b7231bb.jar.sha1

@@ -0,0 +1 @@
+f0ddc3072fd16012dafc74928f87fdfd7669ea4a

plugins/analysis-nori/licenses/lucene-analysis-nori-9.3.0-snapshot-b7231bb.jar.sha1

@@ -0,0 +1 @@
+d761fa983d9c21099c433731d5519651737750c1

plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.3.0-snapshot-b7231bb.jar.sha1

@@ -0,0 +1 @@
+43abbbe7c3c789ac448f898981acf54e487407a6

plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.3.0-snapshot-b7231bb.jar.sha1

@@ -0,0 +1 @@
+55df9442a35fe09d4f3f98bd2dda4d1a1dbfd996

plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.3.0-snapshot-b7231bb.jar.sha1

@@ -0,0 +1 @@
+102cbb1d619b96e1f3e524520658b9327a93aba1

plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.3.0-snapshot-b7231bb.jar.sha1

@@ -0,0 +1 @@
+0d5dc4dfb74d698e51dc9b95268faf6dde4b0815

plugins/discovery-azure-classic/build.gradle

@@ -59,7 +59,7 @@ dependencies {
   api "com.sun.jersey:jersey-client:${versions.jersey}"
   api "com.sun.jersey:jersey-core:${versions.jersey}"
   api "com.sun.jersey:jersey-json:${versions.jersey}"
-  api 'org.codehaus.jettison:jettison:1.4.1'
+  api 'org.codehaus.jettison:jettison:1.5.0'
   api 'com.sun.xml.bind:jaxb-impl:2.2.3-1'
 
   // HACK: javax.xml.bind was removed from default modules in java 9, so we pull the api in here,

plugins/discovery-azure-classic/licenses/jettison-1.5.0.jar.sha1

@@ -0,0 +1 @@
+933c7df7a4b78c9a9322f431014ea699b1fc0cc0

plugins/discovery-gce/build.gradle

@@ -24,7 +24,7 @@ versions << [
 dependencies {
   api "com.google.apis:google-api-services-compute:v1-rev160-${versions.google}"
   api "com.google.api-client:google-api-client:${versions.google}"
-  api "com.google.oauth-client:google-oauth-client:1.34.0"
+  api "com.google.oauth-client:google-oauth-client:1.34.1"
   api "com.google.http-client:google-http-client:${versions.google}"
   api "com.google.http-client:google-http-client-jackson2:${versions.google}"
   api 'com.google.code.findbugs:jsr305:3.0.2'

plugins/discovery-gce/licenses/google-oauth-client-1.34.1.jar.sha1

@@ -0,0 +1 @@
+4a4f88c5e13143f882268c98239fb85c3b2c6cb2

plugins/ingest-attachment/build.gradle

@@ -79,7 +79,7 @@ dependencies {
   api "org.apache.poi:poi:${versions.poi}"
   api "org.apache.poi:poi-ooxml-lite:${versions.poi}"
   api "commons-codec:commons-codec:${versions.commonscodec}"
-  api 'org.apache.xmlbeans:xmlbeans:5.0.3'
+  api 'org.apache.xmlbeans:xmlbeans:5.1.0'
   api 'org.apache.commons:commons-collections4:4.4'
   // MS Office
   api "org.apache.poi:poi-scratchpad:${versions.poi}"

plugins/ingest-attachment/licenses/xmlbeans-5.1.0.jar.sha1

@@ -0,0 +1 @@
+3534ab896663e6f6d8a2cf46882d7407641d7a31

plugins/repository-gcs/build.gradle

@@ -58,7 +58,7 @@ dependencies {
   api 'com.google.cloud:google-cloud-core:2.5.10'
   runtimeOnly 'com.google.guava:guava:30.1.1-jre'
   api 'com.google.guava:failureaccess:1.0.1'
-  api 'com.google.http-client:google-http-client:1.35.0'
+  api 'com.google.http-client:google-http-client:1.42.0'
   api "commons-logging:commons-logging:${versions.commonslogging}"
   api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
   api "commons-codec:commons-codec:${versions.commonscodec}"
@@ -82,7 +82,7 @@ dependencies {
   api 'io.grpc:grpc-context:1.46.0'
   api 'io.opencensus:opencensus-api:0.18.0'
   api 'io.opencensus:opencensus-contrib-http-util:0.18.0'
-  api 'com.google.apis:google-api-services-storage:v1-rev20200814-1.30.10'
+  api 'com.google.apis:google-api-services-storage:v1-rev20220608-1.32.1'
 
   testImplementation project(':test:fixtures:gcs-fixture')
 }

plugins/repository-gcs/licenses/google-api-services-storage-v1-rev20220608-1.32.1.jar.sha1

@@ -0,0 +1 @@
+74724addc6cecac408dad3a6a26423b7647b3724

plugins/repository-gcs/licenses/google-http-client-1.42.0.jar.sha1

@@ -0,0 +1 @@
+4f319ce80ba6888d04a38234916c43d5486842a5

release-notes/opensearch.release-notes-1.3.3.md

@@ -0,0 +1,10 @@
+## Version 1.3.3 Release Notes
+
+### Upgrades
+* Upgrade google-oauth-client to 1.33.3 ([#3502](https://github.com/opensearch-project/OpenSearch/pull/3502))
+* Upgrade log4j-core to 2.17.1 ([#3508](https://github.com/opensearch-project/OpenSearch/pull/3508))
+* Upgrade jdom2 to 2.0.6.1 ([#3509](https://github.com/opensearch-project/OpenSearch/pull/3509))
+
+### Bug Fixes
+* Fixing org.opensearch.monitor.os.OsProbeTests::testLogWarnCpuMessageOnlyOnes when CGroups are not available ([#2101](https://github.com/opensearch-project/OpenSearch/pull/2101))
+* Fixing org.opensearch.monitor.os.OsProbeTests > testLogWarnCpuMessageOnlyOnes when cgroups are available but cgroup stats is not ([#3448](https://github.com/opensearch-project/OpenSearch/pull/3448))

server/licenses/lucene-analysis-common-9.3.0-snapshot-b7231bb.jar.sha1

@@ -0,0 +1 @@
+6c6a9569777e4f01c90ed840e5a04234dfcaf42e

server/licenses/lucene-backward-codecs-9.3.0-snapshot-b7231bb.jar.sha1

@@ -0,0 +1 @@
+a7ef963f9f9f15fc5018c5fa68bae5cf65692ca9

server/licenses/lucene-core-9.3.0-snapshot-b7231bb.jar.sha1

@@ -0,0 +1 @@
+da113c963d62f0c8786d7c294dbbb63d5d7953ab

server/licenses/lucene-grouping-9.3.0-snapshot-b7231bb.jar.sha1

@@ -0,0 +1 @@
+54f65917cfa6c9c54cd0354ba333aa7e0f2980e5

server/licenses/lucene-highlighter-9.3.0-snapshot-b7231bb.jar.sha1

@@ -0,0 +1 @@
+d73ebe32147c9a12d321c0b1273d5e5d797b705f

server/licenses/lucene-join-9.3.0-snapshot-b7231bb.jar.sha1

@@ -0,0 +1 @@
+797c92ffe35af37ab1783906fb93ed95a145a701

server/licenses/lucene-memory-9.3.0-snapshot-b7231bb.jar.sha1

@@ -0,0 +1 @@
+5714d64c39021c65dece8ee979d9ea39a327bb87

server/licenses/lucene-misc-9.3.0-snapshot-b7231bb.jar.sha1

@@ -0,0 +1 @@
+4d401c55114367e574ed51e914661f0a97f91e88

server/licenses/lucene-queries-9.3.0-snapshot-b7231bb.jar.sha1

@@ -0,0 +1 @@
+0f165ff86546565d32a508c82ca80ac2840bcf38

server/licenses/lucene-queryparser-9.3.0-snapshot-b7231bb.jar.sha1

@@ -0,0 +1 @@
+d6fb5af1873628dc026e18b5438042143a9a9824

server/licenses/lucene-sandbox-9.3.0-snapshot-b7231bb.jar.sha1

@@ -0,0 +1 @@
+c48ab8982e6bf9429eded6a06d640db922eb2b69

server/licenses/lucene-spatial-extras-9.3.0-snapshot-b7231bb.jar.sha1

@@ -0,0 +1 @@
+d757dc379fee639f54d0574443c5a6fd0b70613a

server/licenses/lucene-spatial3d-9.3.0-snapshot-b7231bb.jar.sha1

@@ -0,0 +1 @@
+6a4e6de9b40cd027233a3ed00774810c36457a6c

server/licenses/lucene-suggest-9.3.0-snapshot-b7231bb.jar.sha1

@@ -0,0 +1 @@
+e793761c4a4292de0d52f066787ab5f3133382cd

server/src/internalClusterTest/java/org/opensearch/cluster/allocation/AwarenessAllocationIT.java

@@ -45,14 +45,17 @@
 import org.opensearch.cluster.routing.allocation.decider.AwarenessAllocationDecider;
 import org.opensearch.common.Priority;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.test.InternalTestCluster;
 import org.opensearch.test.OpenSearchIntegTestCase;
 import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope;
 
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
+import static org.opensearch.cluster.routing.ShardRoutingState.STARTED;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.empty;
@@ -351,4 +354,140 @@ public void testAwarenessZonesIncrementalNodes() {
         assertThat(counts.get(B_1), equalTo(2));
         assertThat(counts.get(noZoneNode), equalTo(2));
     }
+
+    public void testThreeZoneOneReplicaWithForceZoneValueAndLoadAwareness() throws Exception {
+        int nodeCountPerAZ = 5;
+        int numOfShards = 30;
+        int numOfReplica = 1;
+        Settings commonSettings = Settings.builder()
+            .put("cluster.routing.allocation.awareness.attributes", "zone")
+            .put("cluster.routing.allocation.awareness.force.zone.values", "a,b,c")
+            .put("cluster.routing.allocation.load_awareness.skew_factor", "0.0")
+            .put("cluster.routing.allocation.load_awareness.provisioned_capacity", Integer.toString(nodeCountPerAZ * 3))
+            .build();
+
+        logger.info("--> starting 15 nodes on zones 'a' & 'b' & 'c'");
+        List<String> nodes_in_zone_a = internalCluster().startNodes(
+            nodeCountPerAZ,
+            Settings.builder().put(commonSettings).put("node.attr.zone", "a").build()
+        );
+        List<String> nodes_in_zone_b = internalCluster().startNodes(
+            nodeCountPerAZ,
+            Settings.builder().put(commonSettings).put("node.attr.zone", "b").build()
+        );
+        List<String> nodes_in_zone_c = internalCluster().startNodes(
+            nodeCountPerAZ,
+            Settings.builder().put(commonSettings).put("node.attr.zone", "c").build()
+        );
+
+        // Creating index with 30 primary and 1 replica
+        createIndex(
+            "test-1",
+            Settings.builder()
+                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numOfShards)
+                .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numOfReplica)
+                .build()
+        );
+
+        ClusterHealthResponse health = client().admin()
+            .cluster()
+            .prepareHealth()
+            .setIndices("test-1")
+            .setWaitForEvents(Priority.LANGUID)
+            .setWaitForGreenStatus()
+            .setWaitForNodes(Integer.toString(nodeCountPerAZ * 3))
+            .setWaitForNoRelocatingShards(true)
+            .setWaitForNoInitializingShards(true)
+            .execute()
+            .actionGet();
+        assertFalse(health.isTimedOut());
+
+        ClusterState clusterState = client().admin().cluster().prepareState().execute().actionGet().getState();
+        ObjectIntHashMap<String> counts = new ObjectIntHashMap<>();
+
+        for (IndexRoutingTable indexRoutingTable : clusterState.routingTable()) {
+            for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) {
+                for (ShardRouting shardRouting : indexShardRoutingTable) {
+                    counts.addTo(clusterState.nodes().get(shardRouting.currentNodeId()).getName(), 1);
+                }
+            }
+        }
+
+        assertThat(counts.size(), equalTo(nodeCountPerAZ * 3));
+        // All shards should be started
+        assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(numOfShards * (numOfReplica + 1)));
+
+        // stopping half nodes in zone a
+        int nodesToStop = nodeCountPerAZ / 2;
+        List<Settings> nodeDataPathSettings = new ArrayList<>();
+        for (int i = 0; i < nodesToStop; i++) {
+            nodeDataPathSettings.add(internalCluster().dataPathSettings(nodes_in_zone_a.get(i)));
+            internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodes_in_zone_a.get(i)));
+        }
+
+        client().admin().cluster().prepareReroute().setRetryFailed(true).get();
+        health = client().admin()
+            .cluster()
+            .prepareHealth()
+            .setIndices("test-1")
+            .setWaitForEvents(Priority.LANGUID)
+            .setWaitForNodes(Integer.toString(nodeCountPerAZ * 3 - nodesToStop))
+            .setWaitForNoRelocatingShards(true)
+            .setWaitForNoInitializingShards(true)
+            .execute()
+            .actionGet();
+        assertFalse(health.isTimedOut());
+
+        // Creating another index with 30 primary and 1 replica
+        createIndex(
+            "test-2",
+            Settings.builder()
+                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numOfShards)
+                .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numOfReplica)
+                .build()
+        );
+
+        health = client().admin()
+            .cluster()
+            .prepareHealth()
+            .setIndices("test-1", "test-2")
+            .setWaitForEvents(Priority.LANGUID)
+            .setWaitForNodes(Integer.toString(nodeCountPerAZ * 3 - nodesToStop))
+            .setWaitForNoRelocatingShards(true)
+            .setWaitForNoInitializingShards(true)
+            .execute()
+            .actionGet();
+        assertFalse(health.isTimedOut());
+
+        // Restarting the nodes back
+        for (int i = 0; i < nodesToStop; i++) {
+            internalCluster().startNode(
+                Settings.builder()
+                    .put("node.name", nodes_in_zone_a.get(i))
+                    .put(nodeDataPathSettings.get(i))
+                    .put(commonSettings)
+                    .put("node.attr.zone", "a")
+                    .build()
+            );
+        }
+        client().admin().cluster().prepareReroute().setRetryFailed(true).get();
+
+        health = client().admin()
+            .cluster()
+            .prepareHealth()
+            .setIndices("test-1", "test-2")
+            .setWaitForEvents(Priority.LANGUID)
+            .setWaitForNodes(Integer.toString(nodeCountPerAZ * 3))
+            .setWaitForGreenStatus()
+            .setWaitForActiveShards(2 * numOfShards * (numOfReplica + 1))
+            .setWaitForNoRelocatingShards(true)
+            .setWaitForNoInitializingShards(true)
+            .execute()
+            .actionGet();
+        clusterState = client().admin().cluster().prepareState().execute().actionGet().getState();
+
+        // All shards should be started now and cluster health should be green
+        assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(2 * numOfShards * (numOfReplica + 1)));
+        assertThat(health.isTimedOut(), equalTo(false));
+    }
 }