Add BWC tests for PA

Signed-off-by: Sruti Parthiban <[email protected]>

.github/workflows/gradle.yml

@@ -47,6 +47,12 @@ jobs:
           -Dperformance-analyzer-rca.repo="https://github.com/opensearch-project/performance-analyzer-rca.git" \
           -Dperformance-analyzer-rca.branch=main \
           -Dopensearch.version=1.1.0-SNAPSHOT
+    - name: Assemble PA jar for BWC tests
+      working-directory: ./tmp/performance-analyzer
+      run: |
+        ./gradlew assemble -Dopensearch.version=1.1.0-SNAPSHOT
+        mkdir -p ./src/test/resources/org/opensearch/performanceanalyzer/bwc/1.1.0.0-SNAPSHOT
+        cp ./build/distributions/*.zip ./src/test/resources/org/opensearch/performanceanalyzer/bwc/1.1.0.0-SNAPSHOT
     - name: Generate Jacoco coverage report
       working-directory: ./tmp/performance-analyzer
       run: ./gradlew jacocoTestReport
@@ -58,3 +64,5 @@ jobs:
     - name: Run Integration Tests
       working-directory: ./tmp/performance-analyzer
       run: ./gradlew integTest -Dtests.enableIT -Dtests.useDockerCluster -Dopensearch.version=1.1.0-SNAPSHOT
+    - name: Run PerformanceAnalzyer Backwards Compatibility Tests
+      run: ./gradlew bwcTestSuite -Dtests.security.manager=false

DEVELOPER_GUIDE.md

@@ -28,7 +28,13 @@ To build from the command line, use `./gradlew`.
 ./gradlew clean
 ./gradlew build
 ./gradlew publishToMavenLocal
-```
+```    
+
+`./gradlew spotlessApply` formats code.
+`./gradlew paBwcCluster#mixedClusterTask -Dtests.security.manager=false` launches a cluster with three nodes of bwc version of OpenSearch with performance-analyzer and tests backwards compatibility by upgrading one of the nodes with the current version of OpenSearch with performance-analyzer creating a mixed cluster.
+`./gradlew paBwcCluster#rollingUpgradeClusterTask -Dtests.security.manager=false` launches a cluster with three nodes of bwc version of OpenSearch with performance-analyzer and tests backwards compatibility by performing rolling upgrade of each node with the current version of OpenSearch with performance-analyzer.
+`./gradlew paBwcCluster#fullRestartClusterTask -Dtests.security.manager=false` launches a cluster with three nodes of bwc version of OpenSearch with performance-analyzer and tests backwards compatibility by performing a full restart on the cluster upgrading all the nodes with the current version of OpenSearch with performance-analyzer.
+`./gradlew bwcTestSuite -Dtests.security.manager=false` runs all the above bwc tests combined.
 
 ### Using IntelliJ IDEA
 

build.gradle

@@ -35,6 +35,7 @@ buildscript {
         mavenCentral()
         mavenLocal()
         maven { url "https://plugins.gradle.org/m2/" }
+        maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots/org/opensearch/"}
         jcenter()
     }
 
@@ -115,6 +116,8 @@ licenseHeaders.enabled = true
 validateNebulaPom.enabled = false
 loggerUsageCheck.enabled = false
 
+def _numNodes = findProperty('numNodes') as Integer ?: 1
+
 opensearchplugin {
     name 'opensearch-performance-analyzer'
     description 'OpenSearch Performance Analyzer Plugin'
@@ -300,8 +303,10 @@ gradle.startParameter.excludedTaskNames += [ "forbiddenApisMain",
                                              "thirdPartyAudit",
                                              "testingConventions"]
 
+import java.util.concurrent.Callable
 import org.ajoberstar.gradle.git.tasks.GitClone
 import org.opensearch.gradle.test.RestIntegTestTask
+import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask
 
 static def propEnabled(property) {
     return System.getProperty(property) != null && System.getProperty(property).toLowerCase().equals("true")
@@ -447,6 +452,11 @@ task integTest(type: RestIntegTestTask) {
     onlyIf = {
         propEnabled("tests.enableIT")
     }
+    if (System.getProperty("tests.rest.bwcsuite") == null) {
+        filter {
+            excludeTestsMatching "org.opensearch.performanceanalyzer.bwc.*IT"
+        }
+    }
     if (propEnabled("tests.useDockerCluster")) {
         dependsOn(setupOpenSearchCluster)
     }
@@ -456,6 +466,163 @@ task integTest(type: RestIntegTestTask) {
     classpath = sourceSets.test.runtimeClasspath
 }
 
+testClusters.integTest {
+    testDistribution = "ARCHIVE"
+    // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1
+    if (_numNodes > 1) numberOfNodes = _numNodes
+    // When running integration tests it doesn't forward the --debug-jvm to the cluster anymore
+    // i.e. we have to use a custom property to flag when we want to debug elasticsearch JVM
+    // since we also support multi node integration tests we increase debugPort per node
+    if (System.getProperty("opensearch.debug") != null) {
+        def debugPort = 5005
+        nodes.forEach { node ->
+            node.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=*:${debugPort}")
+            debugPort += 1
+        }
+    }
+    plugin(project.tasks.bundlePlugin.archiveFile)
+}
+
+String bwcVersion = "1.13.0.0"
+String baseName = "paBwcCluster"
+String bwcFilePath = "src/test/resources/org/opensearch/performanceanalyzer/bwc/"
+
+2.times {i ->
+    testClusters {
+        "${baseName}$i" {
+            testDistribution = "ARCHIVE"
+            versions = ["7.10.2","1.1.0-SNAPSHOT"]
+            numberOfNodes = 3
+            plugin(provider(new Callable<RegularFile>(){
+                @Override
+                RegularFile call() throws Exception {
+                    return new RegularFile() {
+                        @Override
+                        File getAsFile() {
+                            return fileTree(bwcFilePath + bwcVersion).getSingleFile()
+                        }
+                    }
+                }
+            }))
+            setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
+            setting 'http.content_type.required', 'true'
+        }
+    }
+}
+
+List<Provider<RegularFile>> plugins = [
+        provider(new Callable<RegularFile>(){
+            @Override
+            RegularFile call() throws Exception {
+                return new RegularFile() {
+                    @Override
+                    File getAsFile() {
+                        return fileTree(bwcFilePath + project.version).getSingleFile()
+                    }
+                }
+            }
+        })
+]
+
+// Creates 2 test clusters with 3 nodes of the old version. 
+2.times {i ->
+    task "${baseName}#oldVersionClusterTask$i"(type: StandaloneRestIntegTestTask) {
+        useCluster testClusters."${baseName}$i"
+        filter {
+            includeTestsMatching "org.opensearch.performanceanalyzer.bwc.*IT"
+        }
+        systemProperty 'tests.rest.bwcsuite', 'old_cluster'
+        systemProperty 'tests.rest.bwcsuite_round', 'old'
+        systemProperty 'tests.plugin_bwc_version', bwcVersion
+        nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}$i".allHttpSocketURI.join(",")}")
+        nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}$i".getName()}")
+    }
+}
+
+// Upgrades one node of the old cluster to new OpenSearch version with upgraded plugin version 
+// This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node.
+// This is also used as a one third upgraded cluster for a rolling upgrade.
+task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) {
+    useCluster testClusters."${baseName}0"
+    dependsOn "${baseName}#oldVersionClusterTask0"
+    doFirst {
+        testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
+    }
+    filter {
+        includeTestsMatching "org.opensearch.performanceanalyzer.bwc.*IT"
+    }
+    systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
+    systemProperty 'tests.rest.bwcsuite_round', 'first'
+    systemProperty 'tests.plugin_bwc_version', bwcVersion
+    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}")
+    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}")
+}
+
+// Upgrades the second node to new OpenSearch version with upgraded plugin version after the first node is upgraded.
+// This results in a mixed cluster with 1 node on the old version and 2 upgraded nodes.
+// This is used for rolling upgrade.
+task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTask) {
+    dependsOn "${baseName}#mixedClusterTask"
+    useCluster testClusters."${baseName}0"
+    doFirst {
+        testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
+    }
+    filter {
+        includeTestsMatching "org.opensearch.performanceanalyzer.bwc.*IT"
+    }
+    systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
+    systemProperty 'tests.rest.bwcsuite_round', 'second'
+    systemProperty 'tests.plugin_bwc_version', bwcVersion
+    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}")
+    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}")
+}
+
+// Upgrades the third node to new OpenSearch version with upgraded plugin version after the second node is upgraded.
+// This results in a fully upgraded cluster.
+// This is used for rolling upgrade.
+task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask) {
+    dependsOn "${baseName}#twoThirdsUpgradedClusterTask"
+    useCluster testClusters."${baseName}0"
+    doFirst {
+        testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
+    }
+    filter {
+        includeTestsMatching "org.opensearch.performanceanalyzer.bwc.*IT"
+    }
+    mustRunAfter "${baseName}#mixedClusterTask"
+    systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
+    systemProperty 'tests.rest.bwcsuite_round', 'third'
+    systemProperty 'tests.plugin_bwc_version', bwcVersion
+    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}")
+    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}")
+}
+
+// Upgrades all the nodes of the old cluster to new OpenSearch version with upgraded plugin version 
+// at the same time resulting in a fully upgraded cluster.
+task "${baseName}#fullRestartClusterTask"(type: StandaloneRestIntegTestTask) {
+    dependsOn "${baseName}#oldVersionClusterTask1"
+    useCluster testClusters."${baseName}1"
+    doFirst {
+        testClusters."${baseName}1".upgradeAllNodesAndPluginsToNextVersion(plugins)
+    }
+    filter {
+        includeTestsMatching "org.opensearch.performanceanalyzer.bwc.*IT"
+    }
+    systemProperty 'tests.rest.bwcsuite', 'upgraded_cluster'
+    systemProperty 'tests.plugin_bwc_version', bwcVersion
+    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}1".allHttpSocketURI.join(",")}")
+    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}1".getName()}")
+}
+
+// A bwc test suite which runs all the bwc tasks combined.
+task bwcTestSuite(type: StandaloneRestIntegTestTask) {
+    exclude '**/*Test*'
+    exclude '**/*IT*'
+    dependsOn tasks.named("${baseName}#mixedClusterTask")
+    dependsOn tasks.named("${baseName}#rollingUpgradeClusterTask")
+    dependsOn tasks.named("${baseName}#fullRestartClusterTask")
+}
+
 // This is afterEvaluate because the bundlePlugin ZIP task is updated afterEvaluate and changes the ZIP name to match the plugin name
 afterEvaluate {
     ospackage {

licenses/performanceanalyzer-rca-1.1.0.0-SNAPSHOT.jar.sha1

@@ -1 +1 @@
-afcf9e4d1413659fdc42fa1a2847c246ffb38c88
+32023ad288f8532bc7fd2ed80e9a152bccef27a6

src/test/java/org/opensearch/performanceanalyzer/ConfigOverridesIT.java

@@ -67,7 +67,7 @@ public void testLegacySimpleOverride() throws Exception {
     }
 
     public void testSimpleOverride(String configOverridesEndpoint) throws Exception {
-        ensurePaAndRcaEnabled();
+        ensurePaAndRcaEnabled(configOverridesEndpoint);
         final ConfigOverrides overrides =
                 getOverrides(
                         Arrays.asList(HOT_SHARD_RCA, HOT_NODE_CLUSTER_RCA),
@@ -115,7 +115,7 @@ public void testLegacyCompositeOverrides() throws Exception {
     }
 
     public void testCompositeOverrides(String configOverridesEndpoint) throws Exception {
-        ensurePaAndRcaEnabled();
+        ensurePaAndRcaEnabled(configOverridesEndpoint);
 
         final ConfigOverrides initialOverrides =
                 getOverrides(

src/test/java/org/opensearch/performanceanalyzer/PerformanceAnalyzerIntegTestBase.java

@@ -63,7 +63,6 @@
 import org.opensearch.common.util.concurrent.ThreadContext;
 import org.opensearch.performanceanalyzer.config.setting.PerformanceAnalyzerClusterSettings;
 import org.opensearch.performanceanalyzer.config.setting.handler.PerformanceAnalyzerClusterSettingHandler;
-import org.opensearch.performanceanalyzer.http_action.config.RestConfig;
 import org.opensearch.performanceanalyzer.util.WaitFor;
 import org.opensearch.test.rest.OpenSearchRestTestCase;
 
@@ -213,26 +212,31 @@ private enum Component {
     }
 
     /**
-     * enableComponent enables PA or RCA on the test cluster
+     * modifyComponent enables/disables PA or RCA on the test cluster
      *
      * @param component Either PA or RCA
      * @return The cluster's {@link Response}
      */
-    public Response enableComponent(Component component) throws Exception {
+    public Response modifyComponent(String base_uri, Component component, boolean enabled)
+            throws Exception {
         String endpoint;
         switch (component) {
             case PA:
-                endpoint = RestConfig.PA_BASE_URI + "/cluster/config";
+                endpoint = base_uri + "/cluster/config";
                 break;
             case RCA:
-                endpoint = RestConfig.PA_BASE_URI + "/rca/cluster/config";
+                endpoint = base_uri + "/rca/cluster/config";
                 break;
             default:
                 throw new IllegalArgumentException(
                         "Unrecognized component value " + component.toString());
         }
         Request request = new Request("POST", endpoint);
-        request.setJsonEntity("{\"enabled\": true}");
+        if (enabled) {
+            request.setJsonEntity("{\"enabled\": true}");
+        } else {
+            request.setJsonEntity("{\"enabled\": false}");
+        }
         return client().performRequest(request);
     }
 
@@ -241,13 +245,13 @@ public Response enableComponent(Component component) throws Exception {
      *
      * @throws Exception If the function is unable to enable PA and RCA
      */
-    public void ensurePaAndRcaEnabled() throws Exception {
+    public void ensurePaAndRcaEnabled(String base_uri) throws Exception {
         // Attempt to enable PA and RCA on the cluster
         WaitFor.waitFor(
                 () -> {
                     try {
-                        Response paResp = enableComponent(Component.PA);
-                        Response rcaResp = enableComponent(Component.RCA);
+                        Response paResp = modifyComponent(base_uri, Component.PA, true);
+                        Response rcaResp = modifyComponent(base_uri, Component.RCA, true);
                         return paResp.getStatusLine().getStatusCode() == HttpStatus.SC_OK
                                 && rcaResp.getStatusLine().getStatusCode() == HttpStatus.SC_OK;
                     } catch (Exception e) {
@@ -258,9 +262,8 @@ public void ensurePaAndRcaEnabled() throws Exception {
                 TimeUnit.MINUTES);
 
         // Sanity check that PA and RCA are enabled on the cluster
-        Response resp =
-                client().performRequest(
-                                new Request("GET", RestConfig.PA_BASE_URI + "/cluster/config"));
+        Response resp = client().performRequest(new Request("GET", base_uri + "/cluster/config"));
+        Assert.assertEquals(resp.getStatusLine().getStatusCode(), HttpStatus.SC_OK);
         Map<String, Object> respMap =
                 mapper.readValue(
                         EntityUtils.toString(resp.getEntity(), "UTF-8"),
@@ -280,6 +283,51 @@ public void ensurePaAndRcaEnabled() throws Exception {
                                         .ordinal()));
     }
 
+    /**
+     * ensurePAStatus makes a best effort to enable/disable PA the test OpenSearch cluster
+     *
+     * @throws Exception If the function is unable to enable PA
+     */
+    public void ensurePAStatus(String base_uri, boolean enabled) throws Exception {
+        // Attempt to enable PA and RCA on the cluster
+        WaitFor.waitFor(
+                () -> {
+                    try {
+                        Response paResp = modifyComponent(base_uri, Component.PA, enabled);
+                        return paResp.getStatusLine().getStatusCode() == HttpStatus.SC_OK;
+                    } catch (Exception e) {
+                        return false;
+                    }
+                },
+                1,
+                TimeUnit.MINUTES);
+        // Wait for cluster to propagate PA state across nodes
+        Thread.sleep(2_000);
+        checkPAEnabled(base_uri, enabled);
+    }
+
+    /**
+     * checkPAEnabled makes a best effort to check if PA is enabled on the test OpenSearch cluster
+     *
+     * @throws Exception If the cluster is unable to enable PA
+     */
+    public void checkPAEnabled(String base_uri, boolean enabled) throws Exception {
+        // Sanity check that PA is enabled on the cluster
+        Response resp = client().performRequest(new Request("GET", base_uri + "/cluster/config"));
+        Assert.assertEquals(resp.getStatusLine().getStatusCode(), HttpStatus.SC_OK);
+        Map<String, Object> respMap =
+                mapper.readValue(
+                        EntityUtils.toString(resp.getEntity(), "UTF-8"),
+                        new TypeReference<Map<String, Object>>() {});
+        Integer state = (Integer) respMap.get("currentPerformanceAnalyzerClusterState");
+        Assert.assertEquals(
+                enabled,
+                PerformanceAnalyzerClusterSettingHandler.checkBit(
+                        state,
+                        PerformanceAnalyzerClusterSettings.PerformanceAnalyzerFeatureBits.PA_BIT
+                                .ordinal()));
+    }
+
     @After
     public void closePaClient() throws Exception {
         OpenSearchRestTestCase.closeClients();