From 499114b5306fc081d3b2f7eb204a2febc149c298 Mon Sep 17 00:00:00 2001 From: naveenpajjuri Date: Mon, 9 May 2022 12:50:08 +0530 Subject: [PATCH 1/2] added basic sanity tests for single cluster scenario. (#393) Signed-off-by: Ankit Kala --- build.gradle | 27 +++++- .../singleCluster/SingleClusterSanityIT.kt | 89 +++++++++++++++++++ 2 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 src/test/kotlin/org/opensearch/replication/singleCluster/SingleClusterSanityIT.kt diff --git a/build.gradle b/build.gradle index 41595157..0c30f286 100644 --- a/build.gradle +++ b/build.gradle @@ -401,6 +401,7 @@ integTest { // We skip BWC test here as those get run as part of separate target `bwcTestSuite`. filter { excludeTestsMatching "org.opensearch.replication.bwc.*IT" + excludeTestsMatching "org.opensearch.replication.singleCluster.SingleClusterSanityIT" } /* @@ -612,7 +613,7 @@ def securityPluginOld = new Callable() { } // We maintain 2 set of clusters here. One for full cluster restart and one for rolling restart + mixed cluster. -List clusters = ["bwcLeader0", "bwcFollower0", "bwcLeader1", "bwcFollower1"] +List clusters = ["bwcLeader0", "bwcFollower0", "bwcLeader1", "bwcFollower1","singleCluster"] // TODO: Make BWC test work with security plugin clusters.each { name -> testClusters { @@ -858,3 +859,27 @@ task "bwcTestSuite"(type: RestIntegTestTask) { dependsOn tasks.named("rollingUpgradeClusterTask") dependsOn tasks.named("fullRestartClusterTask") } + +task integTestSingleCluster(type: RestIntegTestTask) { + useCluster testClusters.singleCluster + doFirst { + getClusters().forEach { cluster -> + String alltransportSocketURI = cluster.nodes.stream().flatMap { node -> + node.getAllTransportPortURI().stream() + }.collect(Collectors.joining(",")) + String allHttpSocketURI = cluster.nodes.stream().flatMap { node -> + node.getAllHttpSocketURI().stream() + }.collect(Collectors.joining(",")) + + systemProperty "tests.cluster.${cluster.name}.http_hosts", "${-> allHttpSocketURI}" + systemProperty "tests.cluster.${cluster.name}.transport_hosts", "${-> alltransportSocketURI}" + systemProperty "tests.cluster.${cluster.name}.security_enabled", "${-> securityEnabled.toString()}" + configureCluster(cluster, securityEnabled) + } + } + + filter { + setIncludePatterns("org.opensearch.replication.singleCluster.SingleClusterSanityIT") + } + nonInputProperties.systemProperty('tests.sanitySingleCluster', "integTestSingleCluster") +} \ No newline at end of file diff --git a/src/test/kotlin/org/opensearch/replication/singleCluster/SingleClusterSanityIT.kt b/src/test/kotlin/org/opensearch/replication/singleCluster/SingleClusterSanityIT.kt new file mode 100644 index 00000000..87f021fa --- /dev/null +++ b/src/test/kotlin/org/opensearch/replication/singleCluster/SingleClusterSanityIT.kt @@ -0,0 +1,89 @@ +package org.opensearch.replication.singleCluster + + +import org.apache.logging.log4j.LogManager +import org.junit.BeforeClass +import org.opensearch.client.ResponseException +import org.opensearch.replication.MultiClusterRestTestCase +import org.opensearch.replication.StartReplicationRequest +import org.opensearch.replication.startReplication +import org.assertj.core.api.Assertions.assertThatThrownBy +import org.junit.Assert +import org.opensearch.replication.stopReplication +import java.util.stream.Collectors + + + + +class SingleClusterSanityIT : MultiClusterRestTestCase() { + + companion object { + private val log = LogManager.getLogger(SingleClusterSanityIT::class.java) + private const val standaloneClusterName = "singleCluster" + private const val REPLICATION_PLUGIN_NAME = "opensearch-cross-cluster-replication" + private const val NUM_NODES = 3 + private const val SAMPLE_INDEX = "sample_test_index" + + @BeforeClass + @JvmStatic + fun setupTestClusters() { + val clusters = HashMap() + clusters.put(standaloneClusterName, createTestCluster(standaloneClusterName, true, true, true, false)) + testClusters = clusters + } + + enum class ClusterState(val value: String) { + SINGLE_CLUSTER_SANITY_SUITE("integTestSingleCluster"); + + companion object { + fun from(s: String): ClusterState? = values().find { it.value == s } + } + } + } + + @Throws(Exception::class) + fun testReplicationPluginWithSingleCluster() { + when(ClusterState.from(System.getProperty("tests.sanitySingleCluster"))) { + ClusterState.SINGLE_CLUSTER_SANITY_SUITE -> basicReplicationSanityWithSingleCluster() + } + } + + fun basicReplicationSanityWithSingleCluster() { + verifyReplicationPluginInstallationOnAllNodes(standaloneClusterName) + VerifyReplicationApis(standaloneClusterName) + } + + @Throws(java.lang.Exception::class) + private fun verifyReplicationPluginInstallationOnAllNodes(clusterName: String) { + val restClient = getClientForCluster(clusterName) + for (i in 0 until NUM_NODES) { + val responseMap = getAsMap(restClient.lowLevelClient, "_nodes/$clusterName-$i/plugins")["nodes"] + as Map>? + Assert.assertTrue(responseMap!!.values.isNotEmpty()) + for (response in responseMap!!.values) { + val plugins = response["plugins"] as List>? + val pluginNames: Set = plugins!!.stream().map { map: Map -> + map["name"] + }.collect(Collectors.toSet()).orEmpty() + Assert.assertTrue(pluginNames.contains(REPLICATION_PLUGIN_NAME)) + } + } + } + + @Throws(java.lang.Exception::class) + private fun VerifyReplicationApis(clusterName: String) { + val follower = getClientForCluster(standaloneClusterName) + assertThatThrownBy { + follower.startReplication( + StartReplicationRequest("sample_connection", SAMPLE_INDEX, SAMPLE_INDEX), + waitForRestore = true + ) + }.isInstanceOf(ResponseException::class.java).hasMessageContaining("no such remote cluster") + assertThatThrownBy { + follower.stopReplication(standaloneClusterName) + }.isInstanceOf(ResponseException::class.java) + .hasMessageContaining("No replication in progress for index:"+standaloneClusterName) + } + +} + From f700040bc2ec272a92d440521a86b6f53efd974a Mon Sep 17 00:00:00 2001 From: ankitkala Date: Tue, 10 May 2022 15:51:06 +0530 Subject: [PATCH 2/2] Add support to run sanity test on remote cluster (#394) Signed-off-by: Ankit Kala --- build.gradle | 22 ++--- scripts/integtest.sh | 86 +++++++++++++++++++ .../replication/MultiClusterRestTestCase.kt | 4 + .../singleCluster/SingleClusterSanityIT.kt | 22 ++--- 4 files changed, 107 insertions(+), 27 deletions(-) create mode 100755 scripts/integtest.sh diff --git a/build.gradle b/build.gradle index 0c30f286..f4508b5d 100644 --- a/build.gradle +++ b/build.gradle @@ -613,7 +613,7 @@ def securityPluginOld = new Callable() { } // We maintain 2 set of clusters here. One for full cluster restart and one for rolling restart + mixed cluster. -List clusters = ["bwcLeader0", "bwcFollower0", "bwcLeader1", "bwcFollower1","singleCluster"] +List clusters = ["bwcLeader0", "bwcFollower0", "bwcLeader1", "bwcFollower1"] // TODO: Make BWC test work with security plugin clusters.each { name -> testClusters { @@ -860,24 +860,12 @@ task "bwcTestSuite"(type: RestIntegTestTask) { dependsOn tasks.named("fullRestartClusterTask") } -task integTestSingleCluster(type: RestIntegTestTask) { - useCluster testClusters.singleCluster +task integTestRemote(type: RestIntegTestTask) { doFirst { - getClusters().forEach { cluster -> - String alltransportSocketURI = cluster.nodes.stream().flatMap { node -> - node.getAllTransportPortURI().stream() - }.collect(Collectors.joining(",")) - String allHttpSocketURI = cluster.nodes.stream().flatMap { node -> - node.getAllHttpSocketURI().stream() - }.collect(Collectors.joining(",")) - - systemProperty "tests.cluster.${cluster.name}.http_hosts", "${-> allHttpSocketURI}" - systemProperty "tests.cluster.${cluster.name}.transport_hosts", "${-> alltransportSocketURI}" - systemProperty "tests.cluster.${cluster.name}.security_enabled", "${-> securityEnabled.toString()}" - configureCluster(cluster, securityEnabled) - } + systemProperty "tests.cluster.follower.http_hosts", System.getProperty("follower.http_host") + systemProperty "tests.cluster.follower.transport_hosts", System.getProperty("follower.transport_host") + systemProperty "tests.cluster.follower.security_enabled", System.getProperty("security_enabled") } - filter { setIncludePatterns("org.opensearch.replication.singleCluster.SingleClusterSanityIT") } diff --git a/scripts/integtest.sh b/scripts/integtest.sh new file mode 100755 index 00000000..9c7fb0a6 --- /dev/null +++ b/scripts/integtest.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +set -e + +function usage() { + echo "" + echo "This script is used to run integration tests for plugin installed on a remote OpenSearch/Dashboards cluster." + echo "--------------------------------------------------------------------------" + echo "Usage: $0 [args]" + echo "" + echo "Required arguments:" + echo "None" + echo "" + echo "Optional arguments:" + echo -e "-b BIND_ADDRESS\t, defaults to localhost | 127.0.0.1, can be changed to any IP or domain name for the cluster location." + echo -e "-p BIND_PORT\t, defaults to 9200, can be changed to any port for the cluster location." + echo -e "-t TRANSPORT_PORT\t, defaults to 9300, can be changed to any port for the cluster location." + echo -e "-s SECURITY_ENABLED\t(true | false), defaults to true. Specify the OpenSearch/Dashboards have security enabled or not." + echo -e "-c CREDENTIAL\t(usename:password), no defaults, effective when SECURITY_ENABLED=true." + echo -e "-h\tPrint this message." + echo "--------------------------------------------------------------------------" +} + +while getopts ":h:b:p:s:c:v:n:t:" arg; do + case $arg in + h) + usage + exit 1 + ;; + b) + BIND_ADDRESS=$OPTARG + ;; + p) + BIND_PORT=$OPTARG + ;; + t) + TRANSPORT_PORT=$OPTARG + ;; + s) + SECURITY_ENABLED=$OPTARG + ;; + c) + CREDENTIAL=$OPTARG + ;; + :) + echo "-${OPTARG} requires an argument" + usage + exit 1 + ;; + ?) + echo "Invalid option: -${OPTARG}" + exit 1 + ;; + esac +done + + +if [ -z "$BIND_ADDRESS" ] +then + BIND_ADDRESS="localhost" +fi + +if [ -z "$BIND_PORT" ] +then + BIND_PORT="9200" +fi + +if [ -z "$TRANSPORT_PORT" ] +then + TRANSPORT_PORT="9300" +fi + +if [ -z "$SECURITY_ENABLED" ] +then + SECURITY_ENABLED="true" +fi + +if [ -z "$CREDENTIAL" ] +then + CREDENTIAL="admin:admin" +fi + +USERNAME=`echo $CREDENTIAL | awk -F ':' '{print $1}'` +PASSWORD=`echo $CREDENTIAL | awk -F ':' '{print $2}'` + +./gradlew integTestRemote -Dfollower.http_host="$BIND_ADDRESS:$BIND_PORT" -Dfollower.transport_host="$BIND_ADDRESS:$TRANSPORT_PORT" -Dsecurity_enabled=$SECURITY_ENABLED -Duser=$USERNAME -Dpassword=$PASSWORD --console=plain diff --git a/src/test/kotlin/org/opensearch/replication/MultiClusterRestTestCase.kt b/src/test/kotlin/org/opensearch/replication/MultiClusterRestTestCase.kt index cda59fc2..65e54c72 100644 --- a/src/test/kotlin/org/opensearch/replication/MultiClusterRestTestCase.kt +++ b/src/test/kotlin/org/opensearch/replication/MultiClusterRestTestCase.kt @@ -454,6 +454,10 @@ abstract class MultiClusterRestTestCase : OpenSearchTestCase() { return OpenSearchRestTestCase.entityAsMap(client.performRequest(Request("GET", endpoint))) } + fun getAsList(client: RestClient, endpoint: String): List { + return OpenSearchRestTestCase.entityAsList(client.performRequest(Request("GET", endpoint))) + } + protected fun createConnectionBetweenClusters(fromClusterName: String, toClusterName: String, connectionName: String="source") { val toCluster = getNamedCluster(toClusterName) val fromCluster = getNamedCluster(fromClusterName) diff --git a/src/test/kotlin/org/opensearch/replication/singleCluster/SingleClusterSanityIT.kt b/src/test/kotlin/org/opensearch/replication/singleCluster/SingleClusterSanityIT.kt index 87f021fa..f7021ccb 100644 --- a/src/test/kotlin/org/opensearch/replication/singleCluster/SingleClusterSanityIT.kt +++ b/src/test/kotlin/org/opensearch/replication/singleCluster/SingleClusterSanityIT.kt @@ -9,6 +9,7 @@ import org.opensearch.replication.StartReplicationRequest import org.opensearch.replication.startReplication import org.assertj.core.api.Assertions.assertThatThrownBy import org.junit.Assert +import org.opensearch.client.Request import org.opensearch.replication.stopReplication import java.util.stream.Collectors @@ -19,16 +20,15 @@ class SingleClusterSanityIT : MultiClusterRestTestCase() { companion object { private val log = LogManager.getLogger(SingleClusterSanityIT::class.java) - private const val standaloneClusterName = "singleCluster" + private const val followerClusterName = "follower" private const val REPLICATION_PLUGIN_NAME = "opensearch-cross-cluster-replication" - private const val NUM_NODES = 3 private const val SAMPLE_INDEX = "sample_test_index" @BeforeClass @JvmStatic fun setupTestClusters() { val clusters = HashMap() - clusters.put(standaloneClusterName, createTestCluster(standaloneClusterName, true, true, true, false)) + clusters.put(followerClusterName, createTestCluster(followerClusterName, true, true, true, false)) testClusters = clusters } @@ -49,15 +49,17 @@ class SingleClusterSanityIT : MultiClusterRestTestCase() { } fun basicReplicationSanityWithSingleCluster() { - verifyReplicationPluginInstallationOnAllNodes(standaloneClusterName) - VerifyReplicationApis(standaloneClusterName) + verifyReplicationPluginInstallationOnAllNodes(followerClusterName) + VerifyReplicationApis(followerClusterName) } @Throws(java.lang.Exception::class) private fun verifyReplicationPluginInstallationOnAllNodes(clusterName: String) { val restClient = getClientForCluster(clusterName) - for (i in 0 until NUM_NODES) { - val responseMap = getAsMap(restClient.lowLevelClient, "_nodes/$clusterName-$i/plugins")["nodes"] + val nodes = getAsList(restClient.lowLevelClient, "_cat/nodes?format=json") as List> + nodes.forEach { node -> + val nodeName = node["name"] + val responseMap = getAsMap(restClient.lowLevelClient, "_nodes/$nodeName/plugins")["nodes"] as Map>? Assert.assertTrue(responseMap!!.values.isNotEmpty()) for (response in responseMap!!.values) { @@ -72,7 +74,7 @@ class SingleClusterSanityIT : MultiClusterRestTestCase() { @Throws(java.lang.Exception::class) private fun VerifyReplicationApis(clusterName: String) { - val follower = getClientForCluster(standaloneClusterName) + val follower = getClientForCluster(followerClusterName) assertThatThrownBy { follower.startReplication( StartReplicationRequest("sample_connection", SAMPLE_INDEX, SAMPLE_INDEX), @@ -80,9 +82,9 @@ class SingleClusterSanityIT : MultiClusterRestTestCase() { ) }.isInstanceOf(ResponseException::class.java).hasMessageContaining("no such remote cluster") assertThatThrownBy { - follower.stopReplication(standaloneClusterName) + follower.stopReplication(followerClusterName) }.isInstanceOf(ResponseException::class.java) - .hasMessageContaining("No replication in progress for index:"+standaloneClusterName) + .hasMessageContaining("No replication in progress for index:"+followerClusterName) } }