Add performance test scripts

jenkins/opensearch/perf-test.jenkinsfile

@@ -0,0 +1,100 @@
+lib = library(identifier: "jenkins@20211118", retriever: legacySCM(scm))
+
+pipeline {
+    agent none
+    environment {
+        AWS_ROLE_ARN = "arn:aws:iam::${AWS_ACCOUNT_PUBLIC}:role/opensearch-test"
+        AWS_ROLE_SESSION_NAME = "jenkins-test-session"
+        BUNDLE_MANIFEST = "bundle-manifest.yml"
+    }
+    tools {
+        jdk "JDK14"
+        maven "maven-3.8.2"
+    }
+    parameters {
+        string(
+            defaultValue: '',
+            name: 'GITHUB_TOKEN',
+            description: 'Github token for account access.',
+            trim: true
+        )
+        string(
+            defaultValue: '',
+            name: 'BUNDLE_MANIFEST_URL',
+            description: 'The bundle manifest URL, e.g. https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.2.2/98/linux/x64/builds/opensearch/manifest.yml.',
+            trim: true
+        )
+        string(
+            defaultValue: '',
+            name: 'AGENT_LABEL',
+            description: 'The agent label where the tests should be executed, e.g. Jenkins-Agent-al2-x64-c54xlarge-Docker-Host.',
+            trim: true
+        )
+        string(
+            defaultValue: '',
+            name: 'TEST_WORKLOAD',
+            description: 'The agent label where the tests should be executed, e.g. Jenkins-Agent-al2-x64-c54xlarge-Docker-Host.',
+            trim: true
+        )
+        string(
+            defaultValue: '',
+            name: 'TEST_ITERATIONS',
+            description: 'The agent label where the tests should be executed, e.g. Jenkins-Agent-al2-x64-c54xlarge-Docker-Host.',
+            trim: true
+        )
+        string(
+            defaultValue: '',
+            name: 'WARMUP_ITERATIONS',
+            description: 'The agent label where the tests should be executed, e.g. Jenkins-Agent-al2-x64-c54xlarge-Docker-Host.',
+            trim: true
+        )
+    }
+
+    stages {
+        stage('perf-test') {
+            agent {
+                node {
+                    label "${AGENT_LABEL}"
+                }
+            }
+            steps {
+                script {
+                    def bundleManifestObj = downloadBuildManifest(
+                        url: BUNDLE_MANIFEST_URL,
+                        path: BUNDLE_MANIFEST
+                    )
+                    String buildId = bundleManifestObj.getArtifactBuildId()
+                    env.BUILD_ID = buildId
+                    env.HAS_SECURITY = bundleManifestObj.components.containsKey("security")
+                    echo "BUNDLE_MANIFEST: ${BUNDLE_MANIFEST}"
+                    echo "BUILD_ID: ${BUILD_ID}"
+                    echo "Security: ${HAS_SECURITY}"
+
+                    runPerfTestScript(bundleManifest: "${BUNDLE_MANIFEST}",
+                        buildId: "${BUILD_ID}",
+                        security: false)
+
+                    if(env.HAS_SECURITY) {
+                     runPerfTestScript(bundleManifest: "${BUNDLE_MANIFEST}",
+                        buildId: "${BUILD_ID}",
+                        security: true)
+                    }
+
+                }
+            }
+
+            post {
+                success {
+                    script {
+                        uploadTestResults(
+                                buildManifestFileName: "${BUNDLE_MANIFEST}",
+                                jobName: 'perf-test',
+                                buildNumber: "${BUILD_ID}"
+                        )
+                    }
+                    cleanWs disableDeferredWipeout: true, deleteDirs: true
+                }
+            }
+        }
+    }
+}

src/run_perf_test.py

@@ -9,6 +9,7 @@
 import argparse
 import os
 import sys
+import time
 
 import yaml
 
@@ -35,6 +36,9 @@ def main():
     parser.add_argument("--bundle-manifest", type=argparse.FileType("r"), help="Bundle Manifest file.", required=True)
     parser.add_argument("--stack", dest="stack", help="Stack name for performance test")
     parser.add_argument("--config", type=argparse.FileType("r"), help="Config file.", required=True)
+    parser.add_argument("--security", dest="security", action="store_true",
+                        help="Security of the cluster should be True/False",
+                        default=False)
     parser.add_argument("--keep", dest="keep", action="store_true", help="Do not delete the working temporary directory.")
     parser.add_argument("--workload", default="nyc_taxis", help="Mensor (internal client) param - Workload name from OpenSeach Benchmark Workloads")
     parser.add_argument("--workload-options", default="{}", help="Mensor (internal client) param - Json object with OpenSearch Benchmark arguments")
@@ -45,13 +49,19 @@ def main():
     manifest = BundleManifest.from_file(args.bundle_manifest)
     config = yaml.safe_load(args.config)
 
+    tests_dir = os.path.join(os.getcwd(), "test-results", "perf-test")
+    os.makedirs(tests_dir, exist_ok=True)
+
     with TemporaryDirectory(keep=args.keep, chdir=True) as work_dir:
         current_workspace = os.path.join(work_dir.name, "infra")
         with GitRepository(get_infra_repo_url(), "main", current_workspace):
-            security = "security" in manifest.components
             with WorkingDirectory(current_workspace):
-                with PerfTestCluster.create(manifest, config, args.stack, security, current_workspace) as (test_cluster_endpoint, test_cluster_port):
-                    perf_test_suite = PerfTestSuite(manifest, test_cluster_endpoint, security, current_workspace, args)
+                with PerfTestCluster.create(manifest, config, args.stack, args.security, current_workspace) as (test_cluster_endpoint, test_cluster_port):
+                    # Stack creation returns control before user-data script execution is complete and the server starts
+                    # Sleep helps with consistent service discovery and test initialization success.
+                    time.sleep(120)
+                    perf_test_suite = PerfTestSuite(manifest, test_cluster_endpoint, args.security,
+                                                    current_workspace, tests_dir, args)
                     perf_test_suite.execute()
 
 

src/test_workflow/perf_test/perf_test_suite.py

@@ -1,15 +1,12 @@
 import os
 import subprocess
 
-from system.working_directory import WorkingDirectory
-
 
 class PerfTestSuite:
     """
     Represents a performance test suite. This class runs rally test on the deployed cluster with the provided IP.
     """
-
-    def __init__(self, bundle_manifest, endpoint, security, current_workspace, args):
+    def __init__(self, bundle_manifest, endpoint, security, current_workspace, test_results_path, args):
         self.manifest = bundle_manifest
         self.work_dir = "mensor/"
         self.endpoint = endpoint
@@ -23,16 +20,24 @@ def __init__(self, bundle_manifest, endpoint, security, current_workspace, args)
             f" --warmup-iters {self.args.warmup_iters} --test-iters {self.args.test_iters}"
         )
 
+        if test_results_path is not None:
+            self.command = (
+                f"pipenv run python test_config.py -i {self.endpoint} -b {self.manifest.build.id}"
+                f" -a {self.manifest.build.architecture} -p {test_results_path}"
+            )
+
+        print(self.command)
+
     def execute(self):
         try:
-            with WorkingDirectory(self.work_dir):
-                dir = os.getcwd()
-                subprocess.check_call("python3 -m pipenv install", cwd=dir, shell=True)
-                subprocess.check_call("pipenv install", cwd=dir, shell=True)
+            os.chdir(os.path.join(self.current_workspace, self.work_dir))
+            dir = os.getcwd()
+            subprocess.check_call("python3 -m pipenv install", cwd=dir, shell=True)
+            subprocess.check_call("pipenv install", cwd=dir, shell=True)
 
-                if self.security:
-                    subprocess.check_call(f"{self.command} -s", cwd=dir, shell=True)
-                else:
-                    subprocess.check_call(f"{self.command}", cwd=dir, shell=True)
+            if self.security:
+                subprocess.check_call(f"{self.command} -s", cwd=dir, shell=True)
+            else:
+                subprocess.check_call(f"{self.command}", cwd=dir, shell=True)
         finally:
             os.chdir(self.current_workspace)

test.sh

@@ -16,8 +16,11 @@ case $1 in
   "bwc-test")
   "$DIR/run.sh" "$DIR/src/run_bwc_test.py" "${@:2}"
   ;;
+  "perf-test")
+  "$DIR/run.sh" "$DIR/src/run_perf_test.py" "${@:2}"
+  ;;
   *)
-  echo "Invalid test suite, run ./test.sh integ-test|bwc-test."
+  echo "Invalid test suite, run ./test.sh integ-test|bwc-test|perf-test."
   exit 1
   ;;
 esac

tests/tests_test_workflow/test_perf_workflow/perf_test/test_perf_test_suite.py

@@ -23,8 +23,10 @@ def setUp(self):
         self.manifest_filename = os.path.join(self.data_path, "bundle_manifest.yml")
         self.manifest = BundleManifest.from_path(self.manifest_filename)
         self.endpoint = None
-        self.perf_test_suite = PerfTestSuite(bundle_manifest=self.manifest, endpoint=None, security=False, current_workspace="current_workspace", args=self.args)
 
+        self.perf_test_suite = PerfTestSuite(bundle_manifest=self.manifest, endpoint=None, security=False,
+                                             current_workspace="current_workspace", test_results_path="test/results/",
+                                             args=self.args)
     def test_execute(self):
         with patch("test_workflow.perf_test.perf_test_suite.os.chdir"):
             with patch("subprocess.check_call") as mock_check_call:

vars/runPerfTestScript.groovy

@@ -0,0 +1,47 @@
+void call(Map args = [:]) {
+    String jobName = args.jobName ?: 'distribution-build-opensearch'
+    lib = library(identifier: 'jenkins@20211123', retriever: legacySCM(scm))
+    def buildManifest = lib.jenkins.BuildManifest.new(readYaml(file: args.bundleManifest))
+    String artifactRootUrl = buildManifest.getArtifactRootUrl(jobName, args.buildId)
+
+    install_npm()
+    install_dependencies()
+    install_opensearch_infra_dependencies()
+    withAWS(role: 'opensearch-test', roleAccount: "${AWS_ACCOUNT_PUBLIC}", duration: 900, roleSessionName: 'jenkins-session') {
+        s3Download(file: "config.yml", bucket: "${ARTIFACT_BUCKET_NAME}", path: "${PERF_TEST_CONFIG_LOCATION}/config.yml", force: true)
+    }
+
+    sh([
+        './test.sh',
+        'perf-test',
+        args.security ? "--stack test-single-security-${args.buildId}" :
+        "--stack test-single-${args.buildId}",
+        "--bundle-manifest ${args.bundleManifest}",
+        "--config config.yml",
+        args.security ? "--security" : ""
+    ].join(' '))
+}
+
+void install_opensearch_infra_dependencies() {
+    sh'''
+        pipenv install "dataclasses_json~=0.5" "aws_requests_auth~=0.4" "json2html~=1.3.0"
+        pipenv install "aws-cdk.core~=1.143.0" "aws_cdk.aws_ec2~=1.143.0" "aws_cdk.aws_iam~=1.143.0"
+        pipenv install "boto3~=1.18" "setuptools~=57.4" "retry~=0.9"
+    '''
+}
+
+void install_npm(){
+    sh'''
+        sudo yum install -y gcc-c++ make
+        curl -sL https://rpm.nodesource.com/setup_16.x | sudo -E bash -
+        sudo yum install -y nodejs --enablerepo=nodesource
+        node -v
+      '''
+}
+
+void install_dependencies() {
+    sh '''
+        sudo npm install -g aws-cdk
+        sudo npm install -g cdk-assume-role-credential-plugin
+    '''
+}