Backport sanity integration tests in remote cluster

build.gradle

@@ -401,6 +401,7 @@ integTest {
     // We skip BWC test here as those get run as part of separate target `bwcTestSuite`.
     filter {
         excludeTestsMatching "org.opensearch.replication.bwc.*IT"
+        excludeTestsMatching "org.opensearch.replication.singleCluster.SingleClusterSanityIT"
     }
 
     /*
@@ -858,3 +859,15 @@ task "bwcTestSuite"(type: RestIntegTestTask) {
     dependsOn tasks.named("rollingUpgradeClusterTask")
     dependsOn tasks.named("fullRestartClusterTask")
 }
+
+task integTestRemote(type: RestIntegTestTask) {
+    doFirst {
+        systemProperty "tests.cluster.follower.http_hosts", System.getProperty("follower.http_host")
+        systemProperty "tests.cluster.follower.transport_hosts", System.getProperty("follower.transport_host")
+        systemProperty "tests.cluster.follower.security_enabled", System.getProperty("security_enabled")
+    }
+    filter {
+        setIncludePatterns("org.opensearch.replication.singleCluster.SingleClusterSanityIT")
+    }
+    nonInputProperties.systemProperty('tests.sanitySingleCluster', "integTestSingleCluster")
+}

scripts/integtest.sh

@@ -0,0 +1,86 @@
+#!/bin/bash
+
+set -e
+
+function usage() {
+    echo ""
+    echo "This script is used to run integration tests for plugin installed on a remote OpenSearch/Dashboards cluster."
+    echo "--------------------------------------------------------------------------"
+    echo "Usage: $0 [args]"
+    echo ""
+    echo "Required arguments:"
+    echo "None"
+    echo ""
+    echo "Optional arguments:"
+    echo -e "-b BIND_ADDRESS\t, defaults to localhost | 127.0.0.1, can be changed to any IP or domain name for the cluster location."
+    echo -e "-p BIND_PORT\t, defaults to 9200, can be changed to any port for the cluster location."
+    echo -e "-t TRANSPORT_PORT\t, defaults to 9300, can be changed to any port for the cluster location."
+    echo -e "-s SECURITY_ENABLED\t(true | false), defaults to true. Specify the OpenSearch/Dashboards have security enabled or not."
+    echo -e "-c CREDENTIAL\t(usename:password), no defaults, effective when SECURITY_ENABLED=true."
+    echo -e "-h\tPrint this message."
+    echo "--------------------------------------------------------------------------"
+}
+
+while getopts ":h:b:p:s:c:v:n:t:" arg; do
+    case $arg in
+        h)
+            usage
+            exit 1
+            ;;
+        b)
+            BIND_ADDRESS=$OPTARG
+            ;;
+        p)
+            BIND_PORT=$OPTARG
+            ;;
+        t)
+            TRANSPORT_PORT=$OPTARG
+            ;;
+        s)
+            SECURITY_ENABLED=$OPTARG
+            ;;
+        c)
+            CREDENTIAL=$OPTARG
+            ;;
+        :)
+            echo "-${OPTARG} requires an argument"
+            usage
+            exit 1
+            ;;
+        ?)
+            echo "Invalid option: -${OPTARG}"
+            exit 1
+            ;;
+    esac
+done
+
+
+if [ -z "$BIND_ADDRESS" ]
+then
+  BIND_ADDRESS="localhost"
+fi
+
+if [ -z "$BIND_PORT" ]
+then
+  BIND_PORT="9200"
+fi
+
+if [ -z "$TRANSPORT_PORT" ]
+then
+  TRANSPORT_PORT="9300"
+fi
+
+if [ -z "$SECURITY_ENABLED" ]
+then
+  SECURITY_ENABLED="true"
+fi
+
+if [ -z "$CREDENTIAL" ]
+then
+  CREDENTIAL="admin:admin"
+fi
+
+USERNAME=`echo $CREDENTIAL | awk -F ':' '{print $1}'`
+PASSWORD=`echo $CREDENTIAL | awk -F ':' '{print $2}'`
+
+./gradlew integTestRemote -Dfollower.http_host="$BIND_ADDRESS:$BIND_PORT" -Dfollower.transport_host="$BIND_ADDRESS:$TRANSPORT_PORT" -Dsecurity_enabled=$SECURITY_ENABLED -Duser=$USERNAME -Dpassword=$PASSWORD --console=plain

src/test/kotlin/org/opensearch/replication/MultiClusterRestTestCase.kt

@@ -454,6 +454,10 @@ abstract class MultiClusterRestTestCase : OpenSearchTestCase() {
         return OpenSearchRestTestCase.entityAsMap(client.performRequest(Request("GET", endpoint)))
     }
 
+    fun getAsList(client: RestClient, endpoint: String): List<Any> {
+        return OpenSearchRestTestCase.entityAsList(client.performRequest(Request("GET", endpoint)))
+    }
+
     protected fun createConnectionBetweenClusters(fromClusterName: String, toClusterName: String, connectionName: String="source") {
         val toCluster = getNamedCluster(toClusterName)
         val fromCluster = getNamedCluster(fromClusterName)

src/test/kotlin/org/opensearch/replication/singleCluster/SingleClusterSanityIT.kt

@@ -0,0 +1,91 @@
+package org.opensearch.replication.singleCluster
+
+
+import org.apache.logging.log4j.LogManager
+import org.junit.BeforeClass
+import org.opensearch.client.ResponseException
+import org.opensearch.replication.MultiClusterRestTestCase
+import org.opensearch.replication.StartReplicationRequest
+import org.opensearch.replication.startReplication
+import org.assertj.core.api.Assertions.assertThatThrownBy
+import org.junit.Assert
+import org.opensearch.client.Request
+import org.opensearch.replication.stopReplication
+import java.util.stream.Collectors
+
+
+
+
+class SingleClusterSanityIT : MultiClusterRestTestCase() {
+
+    companion object {
+        private val log = LogManager.getLogger(SingleClusterSanityIT::class.java)
+        private const val followerClusterName = "follower"
+        private const val REPLICATION_PLUGIN_NAME = "opensearch-cross-cluster-replication"
+        private const val SAMPLE_INDEX = "sample_test_index"
+
+        @BeforeClass
+        @JvmStatic
+        fun setupTestClusters() {
+            val clusters = HashMap<String, TestCluster>()
+            clusters.put(followerClusterName, createTestCluster(followerClusterName, true, true, true, false))
+            testClusters = clusters
+        }
+
+        enum class ClusterState(val value: String) {
+            SINGLE_CLUSTER_SANITY_SUITE("integTestSingleCluster");
+
+            companion object {
+                fun from(s: String): ClusterState? = values().find { it.value == s }
+            }
+        }
+    }
+
+    @Throws(Exception::class)
+    fun testReplicationPluginWithSingleCluster() {
+        when(ClusterState.from(System.getProperty("tests.sanitySingleCluster"))) {
+            ClusterState.SINGLE_CLUSTER_SANITY_SUITE -> basicReplicationSanityWithSingleCluster()
+        }
+    }
+
+    fun basicReplicationSanityWithSingleCluster() {
+        verifyReplicationPluginInstallationOnAllNodes(followerClusterName)
+        VerifyReplicationApis(followerClusterName)
+    }
+
+    @Throws(java.lang.Exception::class)
+    private fun verifyReplicationPluginInstallationOnAllNodes(clusterName: String) {
+        val restClient = getClientForCluster(clusterName)
+        val nodes = getAsList(restClient.lowLevelClient, "_cat/nodes?format=json") as List<Map<String, String>>
+        nodes.forEach { node ->
+            val nodeName = node["name"]
+            val responseMap = getAsMap(restClient.lowLevelClient, "_nodes/$nodeName/plugins")["nodes"]
+                    as Map<String, Map<String, Any>>?
+            Assert.assertTrue(responseMap!!.values.isNotEmpty())
+            for (response in responseMap!!.values) {
+                val plugins = response["plugins"] as List<Map<String, Any>>?
+                val pluginNames: Set<Any?> = plugins!!.stream().map { map: Map<String, Any> ->
+                    map["name"]
+                }.collect(Collectors.toSet()).orEmpty()
+                Assert.assertTrue(pluginNames.contains(REPLICATION_PLUGIN_NAME))
+            }
+        }
+    }
+
+    @Throws(java.lang.Exception::class)
+    private fun VerifyReplicationApis(clusterName: String) {
+        val follower = getClientForCluster(followerClusterName)
+        assertThatThrownBy {
+            follower.startReplication(
+                StartReplicationRequest("sample_connection", SAMPLE_INDEX, SAMPLE_INDEX),
+                waitForRestore = true
+            )
+        }.isInstanceOf(ResponseException::class.java).hasMessageContaining("no such remote cluster")
+        assertThatThrownBy {
+            follower.stopReplication(followerClusterName)
+        }.isInstanceOf(ResponseException::class.java)
+            .hasMessageContaining("No replication in progress for index:"+followerClusterName)
+    }
+
+}
+