Merge branch 'main' into 3.0_bump

.github/workflows/build-and-test.yml

@@ -0,0 +1,51 @@
+name: CCR Test Workflow
+# This workflow is triggered on pull requests to main branch
+on:
+  pull_request:
+    branches:
+      - '*'
+  push:
+    branches:
+      - '*'
+
+# We build for all combinations but run tests only on one combination (linux & latest java)
+jobs:
+  build:
+    continue-on-error: true
+    strategy:
+      matrix:
+        java:
+          - 11
+          - 17
+    # Job name
+    name: Run integration tests on linux with Java ${{ matrix.java }}
+    runs-on: ubuntu-latest
+    steps:
+      # This step uses the setup-java Github action: https://github.com/actions/setup-java
+      - name: Set Up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v1
+        with:
+          java-version: ${{ matrix.java }}
+      # This step uses the checkout Github action: https://github.com/actions/checkout
+      - name: Checkout Branch
+        uses: actions/checkout@v2
+      - name: Build and run Replication tests
+        run: |
+          ./gradlew clean release -D"build.snapshot=true"
+      - name: Upload failed logs
+        uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: logs
+          path: |
+            build/testclusters/integTest-*/logs/*
+            build/testclusters/leaderCluster-*/logs/*
+            build/testclusters/followCluster-*/logs/*
+      - name: Create Artifact Path
+        run: |
+          mkdir -p cross-cluster-replication-artifacts
+          cp ./build/distributions/*.zip cross-cluster-replication-artifacts
+      - name: Uploads coverage
+        with:
+          fetch-depth: 2
+        uses: codecov/[email protected]

.github/workflows/build.yml

@@ -1,4 +1,4 @@
-name: Test and Build Workflow
+name: Build Replication plugin
 # This workflow is triggered on pull requests to main branch
 on:
   pull_request:
@@ -8,16 +8,21 @@ on:
     branches:
       - '*'
 
+# We build for other platforms except linux which is already covered in build-and-test.
+# Also, We're not running tests here as those are already covered with linux build.
 jobs:
   build:
+    continue-on-error: true
     strategy:
       matrix:
         java:
-          - 11
           - 17
+        os:
+          - windows-latest
+          - macos-latest
     # Job name
-    name: Build Replication plugin
-    runs-on: ubuntu-latest
+    name: Java ${{ matrix.java }} On ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
     steps:
       # This step uses the setup-java Github action: https://github.com/actions/setup-java
       - name: Set Up JDK ${{ matrix.java }}
@@ -29,21 +34,4 @@ jobs:
         uses: actions/checkout@v2
       - name: Build and run Replication tests
         run: |
-          ./gradlew clean release -Dbuild.snapshot=true
-      - name: Upload failed logs
-        uses: actions/upload-artifact@v2
-        if: failure()
-        with:
-          name: logs
-          path: |
-            build/testclusters/integTest-*/logs/*
-            build/testclusters/leaderCluster-*/logs/*
-            build/testclusters/followCluster-*/logs/*
-      - name: Create Artifact Path
-        run: |
-          mkdir -p cross-cluster-replication-artifacts
-          cp ./build/distributions/*.zip cross-cluster-replication-artifacts
-      - name: Uploads coverage
-        with:
-          fetch-depth: 2
-        uses: codecov/[email protected]
+          ./gradlew clean release -D"build.snapshot=true" -x test -x IntegTest

build.gradle

@@ -109,7 +109,11 @@ configurations.all {
         force 'com.google.guava:guava:30.1.1-jre'
         force 'com.puppycrawl.tools:checkstyle:8.29'
         force 'commons-codec:commons-codec:1.13'
-        force 'com.fasterxml.jackson.core:jackson-databind:2.13.4'
+        force 'org.apache.httpcomponents:httpclient:4.5.13'
+        force 'org.apache.httpcomponents:httpclient-osgi:4.5.13'
+        force 'org.apache.httpcomponents.client5:httpclient5:5.0.3'
+        force 'org.apache.httpcomponents.client5:httpclient5-osgi:5.0.3'
+        force 'com.fasterxml.jackson.core:jackson-databind:2.13.4.2'
         force 'org.yaml:snakeyaml:1.32'
         force 'org.codehaus.plexus:plexus-utils:3.0.24'
         force 'net.java.dev.jna:jna:5.12.1'
@@ -889,7 +893,7 @@ publishing {
             pom {
                 name = opensearchplugin.name
                 description = opensearchplugin.description
-                group = "org.opensearch.plugin"
+                groupId = "org.opensearch.plugin"
                 licenses {
                     license {
                         name = "The Apache License, Version 2.0"

perf_workflow/requirements.txt

@@ -6,6 +6,7 @@ pipenv
 requests
 retry
 ndg-httpsclient
+psutil
 pyopenssl
 pyasn1
 aws_cdk.core~=1.143.0

perf_workflow/run_perf_suite/ccr_perf_test.py

@@ -52,7 +52,7 @@ def __init__(
         self.execution_id = None
         self.test_succeeded = False
         self.failure_reason = None
-
+        self.use_50_percent_heap = "enable"
     def get_infra_repo_url(self):
         if "GITHUB_TOKEN" in os.environ:
             return "https://${GITHUB_TOKEN}@github.com/opensearch-project/opensearch-infra.git"
@@ -79,6 +79,7 @@ def run(self):
                         self.security,
                         self.test_config.get("DataNodes", 1),
                         self.test_config.get("MasterNodes", 0),
+                        use_50_percent_heap = self.use_50_percent_heap
                     )
                     # TODO: Add support for configurable instance type. Default is m5.2xlarge
                     with self.create_cluster(

src/main/kotlin/org/opensearch/replication/action/index/TransportReplicateIndexAction.kt

@@ -93,7 +93,9 @@ class TransportReplicateIndexAction @Inject constructor(transportService: Transp
                         !leaderSettings.get(ReplicationPlugin.REPLICATED_INDEX_SETTING.key).isNullOrBlank()) {
                     throw IllegalArgumentException("Cannot Replicate a Replicated Index ${request.leaderIndex}")
                 }
-                if (!leaderSettings.getAsBoolean(IndexSettings.INDEX_SOFT_DELETES_SETTING.key, true)) {
+
+                // Soft deletes should be enabled for replication to work.
+                if (!leaderSettings.getAsBoolean(IndexSettings.INDEX_SOFT_DELETES_SETTING.key, false)) {
                     throw IllegalArgumentException("Cannot Replicate an index where the setting ${IndexSettings.INDEX_SOFT_DELETES_SETTING.key} is disabled")
                 }
 
@@ -120,6 +122,7 @@ class TransportReplicateIndexAction @Inject constructor(transportService: Transp
         }
     }
 
+
     private suspend fun getLeaderClusterState(leaderAlias: String, leaderIndex: String): ClusterState {
         val remoteClusterClient = client.getRemoteClusterClient(leaderAlias)
         val clusterStateRequest = remoteClusterClient.admin().cluster().prepareState()
@@ -136,8 +139,18 @@ class TransportReplicateIndexAction @Inject constructor(transportService: Transp
     private suspend fun getLeaderIndexSettings(leaderAlias: String, leaderIndex: String): Settings {
         val remoteClient = client.getRemoteClusterClient(leaderAlias)
         val getSettingsRequest = GetSettingsRequest().includeDefaults(true).indices(leaderIndex)
-        val settingsResponse = remoteClient.suspending(remoteClient.admin().indices()::getSettings,
-                injectSecurityContext = true)(getSettingsRequest)
-        return settingsResponse.indexToSettings.get(leaderIndex) ?: throw IndexNotFoundException("${leaderAlias}:${leaderIndex}")
+        val settingsResponse = remoteClient.suspending(
+            remoteClient.admin().indices()::getSettings,
+            injectSecurityContext = true
+        )(getSettingsRequest)
+
+        val leaderSettings = settingsResponse.indexToSettings.get(leaderIndex)
+            ?: throw IndexNotFoundException("${leaderAlias}:${leaderIndex}")
+        val leaderDefaultSettings = settingsResponse.indexToDefaultSettings.get(leaderIndex)
+            ?: throw IndexNotFoundException("${leaderAlias}:${leaderIndex}")
+
+        // Since we want user configured as well as default settings, we combine both by putting default settings
+        // and then the explicitly set ones to override the default settings.
+        return Settings.builder().put(leaderDefaultSettings).put(leaderSettings).build()
     }
 }