From 719cf938583e224a237f999106cf6026b736fbb1 Mon Sep 17 00:00:00 2001 From: prudhvigodithi Date: Wed, 12 Oct 2022 13:32:53 -0700 Subject: [PATCH 1/5] add group = org.opensearch.plugin Signed-off-by: prudhvigodithi --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 757e6a62b..f18e115a9 100644 --- a/build.gradle +++ b/build.gradle @@ -889,7 +889,7 @@ publishing { pom { name = opensearchplugin.name description = opensearchplugin.description - group = "org.opensearch.plugin" + groupId = "org.opensearch.plugin" licenses { license { name = "The Apache License, Version 2.0" From d02a6a8c4288fb15c1f59d7d3d21c480e3714a71 Mon Sep 17 00:00:00 2001 From: Mohit Kumar <113413713+mohitamg@users.noreply.github.com> Date: Wed, 19 Oct 2022 15:59:39 +0530 Subject: [PATCH 2/5] =?UTF-8?q?Added=20configurable=20parameter=20to=20use?= =?UTF-8?q?=2050%=20of=20heap=20memory=20for=20CCR=20perfor=E2=80=A6=20(#5?= =?UTF-8?q?94)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Added configurable parameter to use 50% of heap memory for CCR performance testing Signed-off-by: Mohit Kumar * Hardcoded the value of use_50_percent_heap and removed it from test_suite.yaml file Signed-off-by: Mohit Kumar Signed-off-by: Mohit Kumar --- perf_workflow/requirements.txt | 1 + perf_workflow/run_perf_suite/ccr_perf_test.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/perf_workflow/requirements.txt b/perf_workflow/requirements.txt index a0e09e61b..616ed43e8 100644 --- a/perf_workflow/requirements.txt +++ b/perf_workflow/requirements.txt @@ -6,6 +6,7 @@ pipenv requests retry ndg-httpsclient +psutil pyopenssl pyasn1 aws_cdk.core~=1.143.0 diff --git a/perf_workflow/run_perf_suite/ccr_perf_test.py b/perf_workflow/run_perf_suite/ccr_perf_test.py index 40185c472..2d8afb516 100644 --- a/perf_workflow/run_perf_suite/ccr_perf_test.py +++ b/perf_workflow/run_perf_suite/ccr_perf_test.py @@ -52,7 +52,7 @@ def __init__( self.execution_id = None self.test_succeeded = False self.failure_reason = None - + self.use_50_percent_heap = "enable" def get_infra_repo_url(self): if "GITHUB_TOKEN" in os.environ: return "https://${GITHUB_TOKEN}@github.com/opensearch-project/opensearch-infra.git" @@ -79,6 +79,7 @@ def run(self): self.security, self.test_config.get("DataNodes", 1), self.test_config.get("MasterNodes", 0), + use_50_percent_heap = self.use_50_percent_heap ) # TODO: Add support for configurable instance type. Default is m5.2xlarge with self.create_cluster( From 0ec98b11c18ae06ab4454cbeb5e9f270891ac297 Mon Sep 17 00:00:00 2001 From: Mohit Kumar <113413713+mohitamg@users.noreply.github.com> Date: Thu, 20 Oct 2022 17:01:30 +0530 Subject: [PATCH 3/5] Updated jackson databind version to 2.13.4.2 (#596) Signed-off-by: Mohit Kumar Signed-off-by: Mohit Kumar --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index f18e115a9..084d6b519 100644 --- a/build.gradle +++ b/build.gradle @@ -113,7 +113,7 @@ configurations.all { force 'org.apache.httpcomponents:httpclient-osgi:4.5.13' force 'org.apache.httpcomponents.client5:httpclient5:5.0.3' force 'org.apache.httpcomponents.client5:httpclient5-osgi:5.0.3' - force 'com.fasterxml.jackson.core:jackson-databind:2.13.4' + force 'com.fasterxml.jackson.core:jackson-databind:2.13.4.2' force 'org.yaml:snakeyaml:1.32' force 'org.codehaus.plexus:plexus-utils:3.0.24' } From d12271a2909f32cf766d2405f7b1d3b8b1746c44 Mon Sep 17 00:00:00 2001 From: Ankit Kala Date: Wed, 26 Oct 2022 12:29:36 +0530 Subject: [PATCH 4/5] Add windows & mac build (#591) Signed-off-by: Ankit Kala Signed-off-by: Ankit Kala --- .github/workflows/build-and-test.yml | 51 ++++++++++++++++++++++++++++ .github/workflows/build.yml | 32 ++++++----------- 2 files changed, 61 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/build-and-test.yml diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml new file mode 100644 index 000000000..e8a6ace59 --- /dev/null +++ b/.github/workflows/build-and-test.yml @@ -0,0 +1,51 @@ +name: CCR Test Workflow +# This workflow is triggered on pull requests to main branch +on: + pull_request: + branches: + - '*' + push: + branches: + - '*' + +# We build for all combinations but run tests only on one combination (linux & latest java) +jobs: + build: + continue-on-error: true + strategy: + matrix: + java: + - 11 + - 17 + # Job name + name: Run integration tests on linux with Java ${{ matrix.java }} + runs-on: ubuntu-latest + steps: + # This step uses the setup-java Github action: https://github.com/actions/setup-java + - name: Set Up JDK ${{ matrix.java }} + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.java }} + # This step uses the checkout Github action: https://github.com/actions/checkout + - name: Checkout Branch + uses: actions/checkout@v2 + - name: Build and run Replication tests + run: | + ./gradlew clean release -D"build.snapshot=true" + - name: Upload failed logs + uses: actions/upload-artifact@v2 + if: failure() + with: + name: logs + path: | + build/testclusters/integTest-*/logs/* + build/testclusters/leaderCluster-*/logs/* + build/testclusters/followCluster-*/logs/* + - name: Create Artifact Path + run: | + mkdir -p cross-cluster-replication-artifacts + cp ./build/distributions/*.zip cross-cluster-replication-artifacts + - name: Uploads coverage + with: + fetch-depth: 2 + uses: codecov/codecov-action@v1.2.1 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c3f599a05..1faf65114 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,4 +1,4 @@ -name: Test and Build Workflow +name: Build Replication plugin # This workflow is triggered on pull requests to main branch on: pull_request: @@ -8,16 +8,21 @@ on: branches: - '*' +# We build for other platforms except linux which is already covered in build-and-test. +# Also, We're not running tests here as those are already covered with linux build. jobs: build: + continue-on-error: true strategy: matrix: java: - - 11 - 17 + os: + - windows-latest + - macos-latest # Job name - name: Build Replication plugin - runs-on: ubuntu-latest + name: Java ${{ matrix.java }} On ${{ matrix.os }} + runs-on: ${{ matrix.os }} steps: # This step uses the setup-java Github action: https://github.com/actions/setup-java - name: Set Up JDK ${{ matrix.java }} @@ -29,21 +34,4 @@ jobs: uses: actions/checkout@v2 - name: Build and run Replication tests run: | - ./gradlew clean release -Dbuild.snapshot=true - - name: Upload failed logs - uses: actions/upload-artifact@v2 - if: failure() - with: - name: logs - path: | - build/testclusters/integTest-*/logs/* - build/testclusters/leaderCluster-*/logs/* - build/testclusters/followCluster-*/logs/* - - name: Create Artifact Path - run: | - mkdir -p cross-cluster-replication-artifacts - cp ./build/distributions/*.zip cross-cluster-replication-artifacts - - name: Uploads coverage - with: - fetch-depth: 2 - uses: codecov/codecov-action@v1.2.1 + ./gradlew clean release -D"build.snapshot=true" -x test -x IntegTest \ No newline at end of file From 67a7073e77d62e1d4b5cb75fc44ff00d06b2a6cc Mon Sep 17 00:00:00 2001 From: Ankit Kala Date: Wed, 26 Oct 2022 17:09:17 +0530 Subject: [PATCH 5/5] Include default index settings during leader setting validation (#601) Signed-off-by: Ankit Kala Signed-off-by: Ankit Kala --- .../index/TransportReplicateIndexAction.kt | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/org/opensearch/replication/action/index/TransportReplicateIndexAction.kt b/src/main/kotlin/org/opensearch/replication/action/index/TransportReplicateIndexAction.kt index 563a99962..80b0e30b5 100644 --- a/src/main/kotlin/org/opensearch/replication/action/index/TransportReplicateIndexAction.kt +++ b/src/main/kotlin/org/opensearch/replication/action/index/TransportReplicateIndexAction.kt @@ -93,7 +93,9 @@ class TransportReplicateIndexAction @Inject constructor(transportService: Transp !leaderSettings.get(ReplicationPlugin.REPLICATED_INDEX_SETTING.key).isNullOrBlank()) { throw IllegalArgumentException("Cannot Replicate a Replicated Index ${request.leaderIndex}") } - if (!leaderSettings.getAsBoolean(IndexSettings.INDEX_SOFT_DELETES_SETTING.key, true)) { + + // Soft deletes should be enabled for replication to work. + if (!leaderSettings.getAsBoolean(IndexSettings.INDEX_SOFT_DELETES_SETTING.key, false)) { throw IllegalArgumentException("Cannot Replicate an index where the setting ${IndexSettings.INDEX_SOFT_DELETES_SETTING.key} is disabled") } @@ -120,6 +122,7 @@ class TransportReplicateIndexAction @Inject constructor(transportService: Transp } } + private suspend fun getLeaderClusterState(leaderAlias: String, leaderIndex: String): ClusterState { val remoteClusterClient = client.getRemoteClusterClient(leaderAlias) val clusterStateRequest = remoteClusterClient.admin().cluster().prepareState() @@ -136,8 +139,18 @@ class TransportReplicateIndexAction @Inject constructor(transportService: Transp private suspend fun getLeaderIndexSettings(leaderAlias: String, leaderIndex: String): Settings { val remoteClient = client.getRemoteClusterClient(leaderAlias) val getSettingsRequest = GetSettingsRequest().includeDefaults(true).indices(leaderIndex) - val settingsResponse = remoteClient.suspending(remoteClient.admin().indices()::getSettings, - injectSecurityContext = true)(getSettingsRequest) - return settingsResponse.indexToSettings.get(leaderIndex) ?: throw IndexNotFoundException("${leaderAlias}:${leaderIndex}") + val settingsResponse = remoteClient.suspending( + remoteClient.admin().indices()::getSettings, + injectSecurityContext = true + )(getSettingsRequest) + + val leaderSettings = settingsResponse.indexToSettings.get(leaderIndex) + ?: throw IndexNotFoundException("${leaderAlias}:${leaderIndex}") + val leaderDefaultSettings = settingsResponse.indexToDefaultSettings.get(leaderIndex) + ?: throw IndexNotFoundException("${leaderAlias}:${leaderIndex}") + + // Since we want user configured as well as default settings, we combine both by putting default settings + // and then the explicitly set ones to override the default settings. + return Settings.builder().put(leaderDefaultSettings).put(leaderSettings).build() } }