From 7affa4d4a410100431344e930d57aec042ba16c6 Mon Sep 17 00:00:00 2001 From: Kunal Kotwani Date: Thu, 24 Feb 2022 10:42:01 -0800 Subject: [PATCH 01/12] Add performance test scripts Signed-off-by: Kunal Kotwani --- jenkins/opensearch/perf-test.jenkinsfile | 100 ++++++++++++++++++ src/run_perf_test.py | 17 ++- .../perf_test/perf_test_suite.py | 28 +++-- test.sh | 5 +- .../perf_test/test_perf_test_suite.py | 3 +- vars/runPerfTestScript.groovy | 47 ++++++++ 6 files changed, 184 insertions(+), 16 deletions(-) create mode 100644 jenkins/opensearch/perf-test.jenkinsfile create mode 100644 vars/runPerfTestScript.groovy diff --git a/jenkins/opensearch/perf-test.jenkinsfile b/jenkins/opensearch/perf-test.jenkinsfile new file mode 100644 index 0000000000..892c1ae1e9 --- /dev/null +++ b/jenkins/opensearch/perf-test.jenkinsfile @@ -0,0 +1,100 @@ +lib = library(identifier: "jenkins@20211118", retriever: legacySCM(scm)) + +pipeline { + agent none + environment { + AWS_ROLE_ARN = "arn:aws:iam::${AWS_ACCOUNT_PUBLIC}:role/opensearch-test" + AWS_ROLE_SESSION_NAME = "jenkins-test-session" + BUNDLE_MANIFEST = "bundle-manifest.yml" + } + tools { + jdk "JDK14" + maven "maven-3.8.2" + } + parameters { + string( + defaultValue: '', + name: 'GITHUB_TOKEN', + description: 'Github token for account access.', + trim: true + ) + string( + defaultValue: '', + name: 'BUNDLE_MANIFEST_URL', + description: 'The bundle manifest URL, e.g. https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.2.2/98/linux/x64/builds/opensearch/manifest.yml.', + trim: true + ) + string( + defaultValue: '', + name: 'AGENT_LABEL', + description: 'The agent label where the tests should be executed, e.g. Jenkins-Agent-al2-x64-c54xlarge-Docker-Host.', + trim: true + ) + string( + defaultValue: '', + name: 'TEST_WORKLOAD', + description: 'The agent label where the tests should be executed, e.g. Jenkins-Agent-al2-x64-c54xlarge-Docker-Host.', + trim: true + ) + string( + defaultValue: '', + name: 'TEST_ITERATIONS', + description: 'The agent label where the tests should be executed, e.g. Jenkins-Agent-al2-x64-c54xlarge-Docker-Host.', + trim: true + ) + string( + defaultValue: '', + name: 'WARMUP_ITERATIONS', + description: 'The agent label where the tests should be executed, e.g. Jenkins-Agent-al2-x64-c54xlarge-Docker-Host.', + trim: true + ) + } + + stages { + stage('perf-test') { + agent { + node { + label "${AGENT_LABEL}" + } + } + steps { + script { + def bundleManifestObj = downloadBuildManifest( + url: BUNDLE_MANIFEST_URL, + path: BUNDLE_MANIFEST + ) + String buildId = bundleManifestObj.getArtifactBuildId() + env.BUILD_ID = buildId + env.HAS_SECURITY = bundleManifestObj.components.containsKey("security") + echo "BUNDLE_MANIFEST: ${BUNDLE_MANIFEST}" + echo "BUILD_ID: ${BUILD_ID}" + echo "Security: ${HAS_SECURITY}" + + runPerfTestScript(bundleManifest: "${BUNDLE_MANIFEST}", + buildId: "${BUILD_ID}", + security: false) + + if(env.HAS_SECURITY) { + runPerfTestScript(bundleManifest: "${BUNDLE_MANIFEST}", + buildId: "${BUILD_ID}", + security: true) + } + + } + } + + post { + success { + script { + uploadTestResults( + buildManifestFileName: "${BUNDLE_MANIFEST}", + jobName: 'perf-test', + buildNumber: "${BUILD_ID}" + ) + } + cleanWs disableDeferredWipeout: true, deleteDirs: true + } + } + } + } +} diff --git a/src/run_perf_test.py b/src/run_perf_test.py index 60100f65bd..03218bf789 100644 --- a/src/run_perf_test.py +++ b/src/run_perf_test.py @@ -9,6 +9,7 @@ import argparse import os import sys +import time import yaml @@ -35,19 +36,29 @@ def main(): parser.add_argument("--bundle-manifest", type=argparse.FileType("r"), help="Bundle Manifest file.", required=True) parser.add_argument("--stack", dest="stack", help="Stack name for performance test") parser.add_argument("--config", type=argparse.FileType("r"), help="Config file.", required=True) + parser.add_argument("--security", dest="security", action="store_true", + help="Security of the cluster should be True/False", + default=False) parser.add_argument("--keep", dest="keep", action="store_true", help="Do not delete the working temporary directory.") args = parser.parse_args() manifest = BundleManifest.from_file(args.bundle_manifest) config = yaml.safe_load(args.config) + tests_dir = os.path.join(os.getcwd(), "test-results", "perf-test") + os.makedirs(tests_dir, exist_ok=True) + with TemporaryDirectory(keep=args.keep, chdir=True) as work_dir: current_workspace = os.path.join(work_dir.name, "infra") with GitRepository(get_infra_repo_url(), "main", current_workspace): - security = "security" in manifest.components with WorkingDirectory(current_workspace): - with PerfTestCluster.create(manifest, config, args.stack, security, current_workspace) as (test_cluster_endpoint, test_cluster_port): - perf_test_suite = PerfTestSuite(manifest, test_cluster_endpoint, security, current_workspace) + with PerfTestCluster.create(manifest, config, args.stack, args.security, current_workspace) \ + as (test_cluster_endpoint, test_cluster_port): + # Stack creation returns control before user-data script execution is complete and the server starts + # Sleep helps with consistent service discovery and test initialization success. + time.sleep(120) + perf_test_suite = PerfTestSuite(manifest, test_cluster_endpoint, args.security, current_workspace, + tests_dir) perf_test_suite.execute() diff --git a/src/test_workflow/perf_test/perf_test_suite.py b/src/test_workflow/perf_test/perf_test_suite.py index 77c34e5176..7963aae743 100644 --- a/src/test_workflow/perf_test/perf_test_suite.py +++ b/src/test_workflow/perf_test/perf_test_suite.py @@ -1,15 +1,13 @@ import os import subprocess -from system.working_directory import WorkingDirectory - class PerfTestSuite: """ Represents a performance test suite. This class runs rally test on the deployed cluster with the provided IP. """ - def __init__(self, bundle_manifest, endpoint, security, current_workspace): + def __init__(self, bundle_manifest, endpoint, security, current_workspace, test_results_path): self.manifest = bundle_manifest self.work_dir = "mensor/" self.endpoint = endpoint @@ -20,16 +18,24 @@ def __init__(self, bundle_manifest, endpoint, security, current_workspace): f" -a {self.manifest.build.architecture} -p {self.current_workspace}" ) + if test_results_path is not None: + self.command = ( + f"pipenv run python test_config.py -i {self.endpoint} -b {self.manifest.build.id}" + f" -a {self.manifest.build.architecture} -p {test_results_path}" + ) + + print(self.command) + def execute(self): try: - with WorkingDirectory(self.work_dir): - dir = os.getcwd() - subprocess.check_call("python3 -m pipenv install", cwd=dir, shell=True) - subprocess.check_call("pipenv install", cwd=dir, shell=True) + os.chdir(os.path.join(self.current_workspace, self.work_dir)) + dir = os.getcwd() + subprocess.check_call("python3 -m pipenv install", cwd=dir, shell=True) + subprocess.check_call("pipenv install", cwd=dir, shell=True) - if self.security: - subprocess.check_call(f"{self.command} -s", cwd=dir, shell=True) - else: - subprocess.check_call(f"{self.command}", cwd=dir, shell=True) + if self.security: + subprocess.check_call(f"{self.command} -s", cwd=dir, shell=True) + else: + subprocess.check_call(f"{self.command}", cwd=dir, shell=True) finally: os.chdir(self.current_workspace) diff --git a/test.sh b/test.sh index 0c2c674dd2..c628e6a1e6 100755 --- a/test.sh +++ b/test.sh @@ -16,8 +16,11 @@ case $1 in "bwc-test") "$DIR/run.sh" "$DIR/src/run_bwc_test.py" "${@:2}" ;; + "perf-test") + "$DIR/run.sh" "$DIR/src/run_perf_test.py" "${@:2}" + ;; *) - echo "Invalid test suite, run ./test.sh integ-test|bwc-test." + echo "Invalid test suite, run ./test.sh integ-test|bwc-test|perf-test." exit 1 ;; esac diff --git a/tests/tests_test_workflow/test_perf_workflow/perf_test/test_perf_test_suite.py b/tests/tests_test_workflow/test_perf_workflow/perf_test/test_perf_test_suite.py index beaba5c2b4..ae725f5f61 100644 --- a/tests/tests_test_workflow/test_perf_workflow/perf_test/test_perf_test_suite.py +++ b/tests/tests_test_workflow/test_perf_workflow/perf_test/test_perf_test_suite.py @@ -18,7 +18,8 @@ def setUp(self): self.manifest_filename = os.path.join(self.data_path, "bundle_manifest.yml") self.manifest = BundleManifest.from_path(self.manifest_filename) self.endpoint = None - self.perf_test_suite = PerfTestSuite(bundle_manifest=self.manifest, endpoint=None, security=False, current_workspace="current_workspace") + self.perf_test_suite = PerfTestSuite(bundle_manifest=self.manifest, endpoint=None, security=False, + current_workspace="current_workspace", test_results_path="test/results/") def test_execute(self): with patch("test_workflow.perf_test.perf_test_suite.os.chdir"): diff --git a/vars/runPerfTestScript.groovy b/vars/runPerfTestScript.groovy new file mode 100644 index 0000000000..1a4fc500c5 --- /dev/null +++ b/vars/runPerfTestScript.groovy @@ -0,0 +1,47 @@ +void call(Map args = [:]) { + String jobName = args.jobName ?: 'distribution-build-opensearch' + lib = library(identifier: 'jenkins@20211123', retriever: legacySCM(scm)) + def buildManifest = lib.jenkins.BuildManifest.new(readYaml(file: args.bundleManifest)) + String artifactRootUrl = buildManifest.getArtifactRootUrl(jobName, args.buildId) + + install_npm() + install_dependencies() + install_opensearch_infra_dependencies() + withAWS(role: 'opensearch-test', roleAccount: "${AWS_ACCOUNT_PUBLIC}", duration: 900, roleSessionName: 'jenkins-session') { + s3Download(file: "config.yml", bucket: "${ARTIFACT_BUCKET_NAME}", path: "${PERF_TEST_CONFIG_LOCATION}/config.yml", force: true) + } + + sh([ + './test.sh', + 'perf-test', + args.security ? "--stack test-single-security-${args.buildId}" : + "--stack test-single-${args.buildId}", + "--bundle-manifest ${args.bundleManifest}", + "--config config.yml", + args.security ? "--security" : "" + ].join(' ')) +} + +void install_opensearch_infra_dependencies() { + sh''' + pipenv install "dataclasses_json~=0.5" "aws_requests_auth~=0.4" "json2html~=1.3.0" + pipenv install "aws-cdk.core~=1.143.0" "aws_cdk.aws_ec2~=1.143.0" "aws_cdk.aws_iam~=1.143.0" + pipenv install "boto3~=1.18" "setuptools~=57.4" "retry~=0.9" + ''' +} + +void install_npm(){ + sh''' + sudo yum install -y gcc-c++ make + curl -sL https://rpm.nodesource.com/setup_16.x | sudo -E bash - + sudo yum install -y nodejs --enablerepo=nodesource + node -v + ''' +} + +void install_dependencies() { + sh ''' + sudo npm install -g aws-cdk + sudo npm install -g cdk-assume-role-credential-plugin + ''' +} \ No newline at end of file From 1347f5c1fbd23589dab982f443eebf01e23b7211 Mon Sep 17 00:00:00 2001 From: Kunal Kotwani Date: Wed, 2 Mar 2022 22:47:43 -0800 Subject: [PATCH 02/12] Update agent to docker Signed-off-by: Kunal Kotwani --- Pipfile | 1 + Pipfile.lock | 148 ++++++++---------- jenkins/opensearch/perf-test.jenkinsfile | 101 +++++++++--- src/run_perf_test.py | 7 +- .../perf_test/perf_test_suite.py | 20 ++- tests/jenkins/TestRunPerfTestScript.groovy | 35 +++++ .../jenkins/data/opensearch-1.3.0-bundle.yml | 16 ++ .../jobs/RunPerfTestScript_Jenkinsfile | 28 ++++ .../jobs/RunPerfTestScript_Jenkinsfile.txt | 26 +++ .../RunPerfTestScriptLibTest.groovy | 62 ++++++++ .../perf_test/test_perf_test_suite.py | 3 +- vars/runPerfTestScript.groovy | 24 ++- 12 files changed, 331 insertions(+), 140 deletions(-) create mode 100644 tests/jenkins/TestRunPerfTestScript.groovy create mode 100644 tests/jenkins/data/opensearch-1.3.0-bundle.yml create mode 100644 tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile create mode 100644 tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile.txt create mode 100644 tests/jenkins/lib-testers/RunPerfTestScriptLibTest.groovy diff --git a/Pipfile b/Pipfile index df9cc4c762..4bf457ba7a 100644 --- a/Pipfile +++ b/Pipfile @@ -16,6 +16,7 @@ pytest = "*" coverage = "~=4.5.4" pytest-cov = "~=2.10.0" jproperties = "~=2.1.1" +retry = "~=0.9" sortedcontainers = "*" cerberus = "~=1.3.4" psutil = "~=5.8" diff --git a/Pipfile.lock b/Pipfile.lock index ee2655bc5a..7835536fec 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "26c25166b1c1c35d0c8871229a8fb34baa37389ba0d5dd40633ac33c8d16e10a" + "sha256": "3da3cb40cf6d9fb381ad4499ae93f01db922e93f1bed2fd233bb880b2ad422cd" }, "pipfile-spec": 6, "requires": { @@ -56,19 +56,19 @@ }, "charset-normalizer": { "hashes": [ - "sha256:876d180e9d7432c5d1dfd4c5d26b72f099d503e8fcc0feb7532c9289be60fcbd", - "sha256:cb957888737fc0bbcd78e3df769addb41fd1ff8cf950dc9e7ad7793f1bf44455" + "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597", + "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df" ], "markers": "python_version >= '3'", - "version": "==2.0.10" + "version": "==2.0.12" }, "click": { "hashes": [ - "sha256:353f466495adaeb40b6b5f592f9f91cb22372351c84caeb068132442a4518ef3", - "sha256:410e932b050f5eed773c4cda94de75971c89cdb3155a72a0831139a79e5ecb5b" + "sha256:6a7a62563bbfabfda3a38f3023a1db4a35978c0abd76f6c9605ecd6554d6d9b1", + "sha256:8458d7b1287c5fb128c90e23381cf99dcde74beaf6c7ff6384ce84d6fe090adb" ], "markers": "python_version >= '3.6'", - "version": "==8.0.3" + "version": "==8.0.4" }, "coverage": { "hashes": [ @@ -125,18 +125,19 @@ }, "distro": { "hashes": [ - "sha256:83f5e5a09f9c5f68f60173de572930effbcc0287bb84fdc4426cb4168c088424", - "sha256:c8713330ab31a034623a9515663ed87696700b55f04556b97c39cd261aa70dc7" + "sha256:151aeccf60c216402932b52e40ee477a939f8d58898927378a02abbe852c1c39", + "sha256:d596311d707e692c2160c37807f83e3820c5d539d5a83e87cfb6babd8ba3a06b" ], - "version": "==1.6.0" + "markers": "python_version >= '3.6'", + "version": "==1.7.0" }, "filelock": { "hashes": [ - "sha256:38b4f4c989f9d06d44524df1b24bd19e167d851f19b50bf3e3559952dddc5b80", - "sha256:cf0fc6a2f8d26bd900f19bf33915ca70ba4dd8c56903eeb14e1e7a2fd7590146" + "sha256:9cd540a9352e432c7246a48fe4e8712b10acb1df2ad1f30e8c070b82ae1fed85", + "sha256:f8314284bfffbdcfa0ff3d7992b023d4c628ced6feb957351d4c48d059f56bc0" ], "markers": "python_version >= '3.7'", - "version": "==3.4.2" + "version": "==3.6.0" }, "flake8": { "hashes": [ @@ -148,11 +149,11 @@ }, "identify": { "hashes": [ - "sha256:6b4b5031f69c48bf93a646b90de9b381c6b5f560df4cbe0ed3cf7650ae741e4d", - "sha256:aa68609c7454dbcaae60a01ff6b8df1de9b39fe6e50b1f6107ec81dcda624aa6" + "sha256:2986942d3974c8f2e5019a190523b0b0e2a07cb8e89bf236727fb4b26f27f8fd", + "sha256:fd906823ed1db23c7a48f9b176a1d71cb8abede1e21ebe614bac7bdd688d9213" ], - "markers": "python_full_version >= '3.6.1'", - "version": "==2.4.4" + "markers": "python_version >= '3.7'", + "version": "==2.4.11" }, "idna": { "hashes": [ @@ -162,14 +163,6 @@ "markers": "python_version >= '3'", "version": "==3.3" }, - "importlib-metadata": { - "hashes": [ - "sha256:92a8b58ce734b2a4494878e0ecf7d79ccd7a128b5fc6014c401e0b61f006f0f6", - "sha256:b7cf7d3fef75f1e4c80a96ca660efbd51473d7e8f39b5ab9210febc7809012a4" - ], - "markers": "python_version < '3.8'", - "version": "==4.10.0" - }, "iniconfig": { "hashes": [ "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3", @@ -257,11 +250,11 @@ }, "platformdirs": { "hashes": [ - "sha256:1d7385c7db91728b83efd0ca99a5afb296cab9d0ed8313a45ed8ba17967ecfca", - "sha256:440633ddfebcc36264232365d7840a970e75e1018d15b4327d11f91909045fda" + "sha256:7535e70dfa32e84d4b34996ea99c5e432fa29a708d0f4e394bbcb2a8faa4f16d", + "sha256:bcae7cab893c2d310a711b70b24efb93334febe65f8de776ee320b517471e227" ], "markers": "python_version >= '3.7'", - "version": "==2.4.1" + "version": "==2.5.1" }, "pluggy": { "hashes": [ @@ -343,19 +336,19 @@ }, "pyparsing": { "hashes": [ - "sha256:04ff808a5b90911829c55c4e26f75fa5ca8a2f5f36aa3a51f68e27033341d3e4", - "sha256:d9bdec0013ef1eb5a84ab39a3b3868911598afa494f5faa038647101504e2b81" + "sha256:18ee9022775d270c55187733956460083db60b37d0d0fb357445f3094eed3eea", + "sha256:a6c06a88f252e6c322f65faf8f418b16213b51bdfaece0524c1c1bc30c63c484" ], "markers": "python_version >= '3.6'", - "version": "==3.0.6" + "version": "==3.0.7" }, "pytest": { "hashes": [ - "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89", - "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134" + "sha256:9ce3ff477af913ecf6321fe337b93a2c0dcf2a0a1439c43f5452112c1e4280db", + "sha256:e30905a0c131d3d94b89624a1cc5afec3e0ba2fbdb151867d8e0ebd49850f171" ], "index": "pypi", - "version": "==6.2.5" + "version": "==7.0.1" }, "pytest-cov": { "hashes": [ @@ -408,6 +401,14 @@ "index": "pypi", "version": "==2.27.1" }, + "retry": { + "hashes": [ + "sha256:ccddf89761fa2c726ab29391837d4327f819ea14d244c232a1d24c67a2f98606", + "sha256:f8bfa8b99b69c4506d6f5bd3b0aabf77f98cdb17f3c9fc3f5ca820033336fba4" + ], + "index": "pypi", + "version": "==0.9.2" + }, "ruyaml": { "hashes": [ "sha256:50e0ee3389c77ad340e209472e0effd41ae0275246df00cdad0a067532171755", @@ -416,6 +417,14 @@ "markers": "python_version >= '3.6'", "version": "==0.91.0" }, + "setuptools": { + "hashes": [ + "sha256:2347b2b432c891a863acadca2da9ac101eae6169b1d3dfee2ec605ecd50dbfe5", + "sha256:e4f30b9f84e5ab3decf945113119649fec09c1fc3507c6ebffec75646c56e62b" + ], + "markers": "python_version >= '3.7'", + "version": "==60.9.3" + }, "six": { "hashes": [ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", @@ -442,36 +451,11 @@ }, "tomli": { "hashes": [ - "sha256:b5bde28da1fed24b9bd1d4d2b8cba62300bfb4ec9a6187a957e8ddb9434c5224", - "sha256:c292c34f58502a1eb2bbb9f5bbc9a5ebc37bee10ffb8c2d6bbdfa8eb13cc14e1" + "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", + "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f" ], "markers": "python_version >= '3.7'", - "version": "==2.0.0" - }, - "typed-ast": { - "hashes": [ - "sha256:24058827d8f5d633f97223f5148a7d22628099a3d2efe06654ce872f46f07cdb", - "sha256:256115a5bc7ea9e665c6314ed6671ee2c08ca380f9d5f130bd4d2c1f5848d695", - "sha256:38cf5c642fa808300bae1281460d4f9b7617cf864d4e383054a5ef336e344d32", - "sha256:484137cab8ecf47e137260daa20bafbba5f4e3ec7fda1c1e69ab299b75fa81c5", - "sha256:4f30a2bcd8e68adbb791ce1567fdb897357506f7ea6716f6bbdd3053ac4d9471", - "sha256:591bc04e507595887160ed7aa8d6785867fb86c5793911be79ccede61ae96f4d", - "sha256:5b6ab14c56bc9c7e3c30228a0a0b54b915b1579613f6e463ba6f4eb1382e7fd4", - "sha256:5d8314c92414ce7481eee7ad42b353943679cf6f30237b5ecbf7d835519e1212", - "sha256:71dcda943a471d826ea930dd449ac7e76db7be778fcd722deb63642bab32ea3f", - "sha256:7c42707ab981b6cf4b73490c16e9d17fcd5227039720ca14abe415d39a173a30", - "sha256:9caaf2b440efb39ecbc45e2fabde809cbe56272719131a6318fd9bf08b58e2cb", - "sha256:a2b8d7007f6280e36fa42652df47087ac7b0a7d7f09f9468f07792ba646aac2d", - "sha256:a6d495c1ef572519a7bac9534dbf6d94c40e5b6a608ef41136133377bba4aa08", - "sha256:a80d84f535642420dd17e16ae25bb46c7f4c16ee231105e7f3eb43976a89670a", - "sha256:b53ae5de5500529c76225d18eeb060efbcec90ad5e030713fe8dab0fb4531631", - "sha256:b6d17f37f6edd879141e64a5db17b67488cfeffeedad8c5cec0392305e9bc775", - "sha256:c9bcad65d66d594bffab8575f39420fe0ee96f66e23c4d927ebb4e24354ec1af", - "sha256:ca9e8300d8ba0b66d140820cf463438c8e7b4cdc6fd710c059bfcfb1531d03fb", - "sha256:de4ecae89c7d8b56169473e08f6bfd2df7f95015591f43126e4ea7865928677e" - ], - "markers": "python_version < '3.8'", - "version": "==1.5.1" + "version": "==2.0.1" }, "types-pyyaml": { "hashes": [ @@ -483,26 +467,26 @@ }, "types-requests": { "hashes": [ - "sha256:2e0e100dd489f83870d4f61949d3a7eae4821e7bfbf46c57e463c38f92d473d4", - "sha256:f38bd488528cdcbce5b01dc953972f3cead0d060cfd9ee35b363066c25bab13c" + "sha256:506279bad570c7b4b19ac1f22e50146538befbe0c133b2cea66a9b04a533a859", + "sha256:6a7ed24b21780af4a5b5e24c310b2cd885fb612df5fd95584d03d87e5f2a195a" ], "index": "pypi", - "version": "==2.27.7" + "version": "==2.27.11" }, "types-urllib3": { "hashes": [ - "sha256:3adcf2cb5981809091dbff456e6999fe55f201652d8c360f99997de5ac2f556e", - "sha256:cfd1fbbe4ba9a605ed148294008aac8a7b8b7472651d1cc357d507ae5962e3d2" + "sha256:a26898f530e6c3f43f25b907f2b884486868ffd56a9faa94cbf9b3eb6e165d6a", + "sha256:d755278d5ecd7a7a6479a190e54230f241f1a99c19b81518b756b19dc69e518c" ], - "version": "==1.26.7" + "version": "==1.26.10" }, "typing-extensions": { "hashes": [ - "sha256:4ca091dea149f945ec56afb48dae714f21e8692ef22a395223bcd328961b6a0e", - "sha256:7f001e5ac290a0c0401508864c7ec868be4e701886d5b573a9528ed3973d9d3b" + "sha256:1a9462dcc3347a79b1f1c0271fbe79e844580bb598bafa1ed208b94da3cdcd42", + "sha256:21c85e0fe4b9a155d0799430b0ad741cdce7e359660ccbd8b530613e8df88ce2" ], - "markers": "python_version < '3.8'", - "version": "==4.0.1" + "markers": "python_version >= '3.6'", + "version": "==4.1.1" }, "urllib3": { "hashes": [ @@ -522,19 +506,19 @@ }, "virtualenv": { "hashes": [ - "sha256:339f16c4a86b44240ba7223d0f93a7887c3ca04b5f9c8129da7958447d079b09", - "sha256:d8458cf8d59d0ea495ad9b34c2599487f8a7772d796f9910858376d1600dd2dd" + "sha256:dd448d1ded9f14d1a4bfa6bfc0c5b96ae3be3f2d6c6c159b23ddcfd701baa021", + "sha256:e9dd1a1359d70137559034c0f5433b34caf504af2dc756367be86a5a32967134" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==20.13.0" + "version": "==20.13.3" }, "yamlfix": { "hashes": [ - "sha256:66094ed651d598baff809cfe9d0fdce447b70ad844c4319c18913a5ff58721af", - "sha256:b3d32734506bfbe28c4edea05f593e4a3a8ea4c1ad9b3490e971b0237ab6188d" + "sha256:3ea6ae93d68d1c4ecafaddd2a53870b6da0ff657eb34d7b78ff40ea485b15430", + "sha256:c8d5497b7b8733c21e215fd64510d96f5363bcedbdc1660ee56311a171d7fef5" ], "index": "pypi", - "version": "==0.8.0" + "version": "==0.8.2" }, "yamllint": { "hashes": [ @@ -542,14 +526,6 @@ ], "index": "pypi", "version": "==1.26.3" - }, - "zipp": { - "hashes": [ - "sha256:9f50f446828eb9d45b267433fd3e9da8d801f614129124863f9c51ebceafb87d", - "sha256:b47250dd24f92b7dd6a0a8fc5244da14608f3ca90a5efcd37a3b1642fac9a375" - ], - "markers": "python_version >= '3.7'", - "version": "==3.7.0" } }, "develop": {} diff --git a/jenkins/opensearch/perf-test.jenkinsfile b/jenkins/opensearch/perf-test.jenkinsfile index 892c1ae1e9..918b9a2579 100644 --- a/jenkins/opensearch/perf-test.jenkinsfile +++ b/jenkins/opensearch/perf-test.jenkinsfile @@ -2,48 +2,39 @@ lib = library(identifier: "jenkins@20211118", retriever: legacySCM(scm)) pipeline { agent none + options { + timeout(time: 10, unit: 'HOURS') + } environment { - AWS_ROLE_ARN = "arn:aws:iam::${AWS_ACCOUNT_PUBLIC}:role/opensearch-test" - AWS_ROLE_SESSION_NAME = "jenkins-test-session" + AGENT_X64 = 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' + AGENT_IMAGE = 'opensearchstaging/ci-runner:ci-runner-centos7-v1' BUNDLE_MANIFEST = "bundle-manifest.yml" } - tools { - jdk "JDK14" - maven "maven-3.8.2" - } parameters { string( - defaultValue: '', name: 'GITHUB_TOKEN', description: 'Github token for account access.', trim: true ) string( - defaultValue: '', name: 'BUNDLE_MANIFEST_URL', description: 'The bundle manifest URL, e.g. https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.2.2/98/linux/x64/builds/opensearch/manifest.yml.', trim: true ) string( - defaultValue: '', - name: 'AGENT_LABEL', - description: 'The agent label where the tests should be executed, e.g. Jenkins-Agent-al2-x64-c54xlarge-Docker-Host.', - trim: true - ) - string( - defaultValue: '', + defaultValue: 'nyc_taxis', name: 'TEST_WORKLOAD', description: 'The agent label where the tests should be executed, e.g. Jenkins-Agent-al2-x64-c54xlarge-Docker-Host.', trim: true ) string( - defaultValue: '', + defaultValue: '1', name: 'TEST_ITERATIONS', description: 'The agent label where the tests should be executed, e.g. Jenkins-Agent-al2-x64-c54xlarge-Docker-Host.', trim: true ) string( - defaultValue: '', + defaultValue: '0', name: 'WARMUP_ITERATIONS', description: 'The agent label where the tests should be executed, e.g. Jenkins-Agent-al2-x64-c54xlarge-Docker-Host.', trim: true @@ -51,10 +42,26 @@ pipeline { } stages { + stage('validate-parameters') { + steps { + script { + if (BUNDLE_MANIFEST_URL == '') { + currentBuild.result = 'ABORTED' + error("Performance Tests failed to start. Missing parameter: BUNDLE_MANIFEST_URL.") + } + if (GITHUB_TOKEN == '') { + currentBuild.result = 'ABORTED' + error("Performance Tests failed to start. Missing parameter: GITHUB_TOKEN.") + } + } + } + } stage('perf-test') { agent { - node { - label "${AGENT_LABEL}" + docker { + label AGENT_X64 + image AGENT_IMAGE + alwaysPull true } } steps { @@ -70,14 +77,25 @@ pipeline { echo "BUILD_ID: ${BUILD_ID}" echo "Security: ${HAS_SECURITY}" + lib.jenkins.Messages.new(this).add('perf-test', "[Test]Executing performance tests for #${BUILD_ID}") + runPerfTestScript(bundleManifest: "${BUNDLE_MANIFEST}", buildId: "${BUILD_ID}", - security: false) + security: false, + workload: TEST_WORKLOAD, + testIterations: TEST_ITERATIONS, + warmupIterations: WARMUP_ITERATIONS) + + lib.jenkins.Messages.new(this).add('perf-test', "[Test]Performance tests without security for ${BUILD_ID} completed") if(env.HAS_SECURITY) { - runPerfTestScript(bundleManifest: "${BUNDLE_MANIFEST}", - buildId: "${BUILD_ID}", - security: true) + runPerfTestScript(bundleManifest: "${BUNDLE_MANIFEST}", + buildId: "${BUILD_ID}", + security: true, + workload: TEST_WORKLOAD, + testIterations: TEST_ITERATIONS, + warmupIterations: WARMUP_ITERATIONS) + lib.jenkins.Messages.new(this).add('perf-test', "[Test]Performance tests with security for ${BUILD_ID} completed") } } @@ -92,7 +110,42 @@ pipeline { buildNumber: "${BUILD_ID}" ) } - cleanWs disableDeferredWipeout: true, deleteDirs: true + } + always { + postCleanup() + } + } + } + } + + post { + success { + node(AGENT_X64) { + script { + def stashed = lib.jenkins.Messages.new(this).get(['perf-test']) + publishNotification( + icon: ':white_check_mark:', + message: 'Performance Tests Successful', + extra: stashed, + credentialsId: 'INTEG_TEST_WEBHOOK', + ) + + postCleanup() + } + } + } + failure { + node(AGENT_X64) { + script { + def stashed = lib.jenkins.Messages.new(this).get(['perf-test']) + publishNotification( + icon: ':warning:', + message: 'Failed Performance Tests', + extra: stashed, + credentialsId: 'INTEG_TEST_WEBHOOK', + ) + + postCleanup() } } } diff --git a/src/run_perf_test.py b/src/run_perf_test.py index 1efaa41283..8907803632 100644 --- a/src/run_perf_test.py +++ b/src/run_perf_test.py @@ -9,9 +9,9 @@ import argparse import os import sys -import time import yaml +from retry.api import retry_call from git.git_repository import GitRepository from manifests.bundle_manifest import BundleManifest @@ -57,12 +57,9 @@ def main(): with GitRepository(get_infra_repo_url(), "main", current_workspace): with WorkingDirectory(current_workspace): with PerfTestCluster.create(manifest, config, args.stack, args.security, current_workspace) as (test_cluster_endpoint, test_cluster_port): - # Stack creation returns control before user-data script execution is complete and the server starts - # Sleep helps with consistent service discovery and test initialization success. - time.sleep(120) perf_test_suite = PerfTestSuite(manifest, test_cluster_endpoint, args.security, current_workspace, tests_dir, args) - perf_test_suite.execute() + retry_call(perf_test_suite.execute, tries=3, delay=60, backoff=2) if __name__ == "__main__": diff --git a/src/test_workflow/perf_test/perf_test_suite.py b/src/test_workflow/perf_test/perf_test_suite.py index eedec4bad4..b551d509d5 100644 --- a/src/test_workflow/perf_test/perf_test_suite.py +++ b/src/test_workflow/perf_test/perf_test_suite.py @@ -1,6 +1,8 @@ import os import subprocess +from system.working_directory import WorkingDirectory + class PerfTestSuite: """ @@ -26,18 +28,14 @@ def __init__(self, bundle_manifest, endpoint, security, current_workspace, test_ f" -a {self.manifest.build.architecture} -p {test_results_path}" ) - print(self.command) - def execute(self): try: - os.chdir(os.path.join(self.current_workspace, self.work_dir)) - dir = os.getcwd() - subprocess.check_call("python3 -m pipenv install", cwd=dir, shell=True) - subprocess.check_call("pipenv install", cwd=dir, shell=True) - - if self.security: - subprocess.check_call(f"{self.command} -s", cwd=dir, shell=True) - else: - subprocess.check_call(f"{self.command}", cwd=dir, shell=True) + current_workspace = os.path.join(self.current_workspace, self.work_dir) + with WorkingDirectory(current_workspace): + subprocess.check_call("pipenv install", cwd=current_workspace, shell=True) + if self.security: + subprocess.check_call(f"{self.command} -s", cwd=current_workspace, shell=True) + else: + subprocess.check_call(f"{self.command}", cwd=current_workspace, shell=True) finally: os.chdir(self.current_workspace) diff --git a/tests/jenkins/TestRunPerfTestScript.groovy b/tests/jenkins/TestRunPerfTestScript.groovy new file mode 100644 index 0000000000..1fbfa10a89 --- /dev/null +++ b/tests/jenkins/TestRunPerfTestScript.groovy @@ -0,0 +1,35 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +import jenkins.tests.BuildPipelineTest +import org.junit.Before +import org.junit.Test + + +class TestRunPerfTestScript extends BuildPipelineTest { + + @Before + void setUp() { + this.registerLibTester(new RunPerfTestScriptLibTester( + 'dummy_job', + 'tests/jenkins/data/opensearch-1.3.0-bundle.yml', + '1236', + 'false', + 'nyc_taxis', + '1', + '1' + ) + ) + super.setUp() + } + + @Test + public void TestRunPerfTestScript() { + super.testPipeline("tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile") + } +} diff --git a/tests/jenkins/data/opensearch-1.3.0-bundle.yml b/tests/jenkins/data/opensearch-1.3.0-bundle.yml new file mode 100644 index 0000000000..8a70c63e74 --- /dev/null +++ b/tests/jenkins/data/opensearch-1.3.0-bundle.yml @@ -0,0 +1,16 @@ +--- +schema-version: '1.1' +build: + name: OpenSearch + version: 1.3.0 + platform: linux + architecture: x64 + distribution: tar + location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/1236/linux/x64/dist/opensearch/opensearch-1.3.0-linux-x64.tar.gz + id: '1236' +components: + - name: OpenSearch + repository: https://github.com/opensearch-project/OpenSearch.git + ref: 1.x + commit_id: 22408088f002a4fc8cdd3b2ed7438866c14c5069 + location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/1236/linux/x64/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64.tar.gz diff --git a/tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile b/tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile new file mode 100644 index 0000000000..a3261db69b --- /dev/null +++ b/tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile @@ -0,0 +1,28 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +pipeline { + agent none + stages { + stage('perf-test') { + steps { + script { + runPerfTestScript( + jobName: 'dummy_job', + bundleManifest: 'tests/jenkins/data/opensearch-1.3.0-bundle.yml', + buildId: '1236', + security: 'false', + workload: 'nyc_taxis', + testIterations: '1', + warmupIterations: '1' + ) + } + } + } + } +} \ No newline at end of file diff --git a/tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile.txt b/tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile.txt new file mode 100644 index 0000000000..2b58cb26e5 --- /dev/null +++ b/tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile.txt @@ -0,0 +1,26 @@ + RunPerfTestScript_Jenkinsfile.run() + RunPerfTestScript_Jenkinsfile.pipeline(groovy.lang.Closure) + RunPerfTestScript_Jenkinsfile.echo(Executing on agent [label:none]) + RunPerfTestScript_Jenkinsfile.stage(perf-test, groovy.lang.Closure) + RunPerfTestScript_Jenkinsfile.script(groovy.lang.Closure) + RunPerfTestScript_Jenkinsfile.runPerfTestScript({jobName=dummy_job, bundleManifest=tests/jenkins/data/opensearch-1.3.0-bundle.yml, buildId=1236, security=false, workload=nyc_taxis, testIterations=1, warmupIterations=1}) + runPerfTestScript.legacySCM(groovy.lang.Closure) + runPerfTestScript.library({identifier=jenkins@20211123, retriever=null}) + runPerfTestScript.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) + BuildManifest.asBoolean() + BuildManifest.getArtifactRootUrl(dummy_job, 1236) + runPerfTestScript.sh( + npm install -g fs-extra + npm install -g chalk@4.1.2 + npm install -g @aws-cdk/cloudformation-diff + npm install -g aws-cdk + npm install -g cdk-assume-role-credential-plugin@1.4.0 + ) + runPerfTestScript.sh( + pipenv install "dataclasses_json~=0.5" "aws_requests_auth~=0.4" "json2html~=1.3.0" + pipenv install "aws-cdk.core~=1.143.0" "aws_cdk.aws_ec2~=1.143.0" "aws_cdk.aws_iam~=1.143.0" + pipenv install "boto3~=1.18" "setuptools~=57.4" "retry~=0.9" + ) + runPerfTestScript.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + runPerfTestScript.s3Download({file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true}) + runPerfTestScript.sh(./test.sh perf-test --stack test-single-security-1236 --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config config.yml --security --workload nyc_taxis --test-iters 1 --warmup-iters 1) diff --git a/tests/jenkins/lib-testers/RunPerfTestScriptLibTest.groovy b/tests/jenkins/lib-testers/RunPerfTestScriptLibTest.groovy new file mode 100644 index 0000000000..6eed917a93 --- /dev/null +++ b/tests/jenkins/lib-testers/RunPerfTestScriptLibTest.groovy @@ -0,0 +1,62 @@ +import static org.hamcrest.CoreMatchers.notNullValue +import static org.hamcrest.MatcherAssert.assertThat + + +class RunPerfTestScriptLibTester extends LibFunctionTester { + + private String jobName + private String bundleManifest + private String buildId + private String security + private String workload + private String testIterations + private String warmupIterations + + public RunPerfTestScriptLibTester(jobName, bundleManifest, buildId, security, workload, testIterations, warmupIterations){ + this.jobName = jobName + this.bundleManifest = bundleManifest + this.buildId = buildId + this.security = security + this.workload = workload + this.testIterations = testIterations + this.warmupIterations = warmupIterations + } + + void configure(helper, binding) { + helper.registerAllowedMethod("s3Download", [Map]) + helper.registerAllowedMethod("withAWS", [Map, Closure], { args, closure -> + closure.delegate = delegate + return helper.callClosure(closure) + }) + + binding.setVariable('GITHUB_TOKEN', 'test_token') + binding.setVariable('ARTIFACT_BUCKET_NAME', 'test_bucket') + binding.setVariable('PERF_TEST_CONFIG_LOCATION', 'test_config') + binding.setVariable('ARTIFACT_DOWNLOAD_ROLE_NAME', 'Dummy_Download_Role') + binding.setVariable('AWS_ACCOUNT_PUBLIC', 'dummy_account') + + } + + void parameterInvariantsAssertions(call) { + assertThat(call.args.bundleManifest.first(), notNullValue()) + assertThat(call.args.buildId.first(), notNullValue()) + assertThat(call.args.security.first(), notNullValue()) + assertThat(call.args.workload.first(), notNullValue()) + assertThat(call.args.testIterations.first(), notNullValue()) + assertThat(call.args.warmupIterations.first(), notNullValue()) + } + + boolean expectedParametersMatcher(call) { + return call.args.jobName.first().toString().equals(this.jobName) + && call.args.bundleManifest.first().toString().equals(this.bundleManifest) + && call.args.buildId.first().toString().equals(this.buildId) + && call.args.security.first().toString().equals(this.security) + && call.args.workload.first().toString().equals(this.workload) + && call.args.testIterations.first().toString().equals(this.testIterations) + && call.args.warmupIterations.first().toString().equals(this.warmupIterations) + } + + String libFunctionName() { + return 'runPerfTestScript' + } +} diff --git a/tests/tests_test_workflow/test_perf_workflow/perf_test/test_perf_test_suite.py b/tests/tests_test_workflow/test_perf_workflow/perf_test/test_perf_test_suite.py index 957fe922bb..35e244485f 100644 --- a/tests/tests_test_workflow/test_perf_workflow/perf_test/test_perf_test_suite.py +++ b/tests/tests_test_workflow/test_perf_workflow/perf_test/test_perf_test_suite.py @@ -27,8 +27,9 @@ def setUp(self): self.perf_test_suite = PerfTestSuite(bundle_manifest=self.manifest, endpoint=None, security=False, current_workspace="current_workspace", test_results_path="test/results/", args=self.args) + def test_execute(self): with patch("test_workflow.perf_test.perf_test_suite.os.chdir"): with patch("subprocess.check_call") as mock_check_call: self.perf_test_suite.execute() - self.assertEqual(mock_check_call.call_count, 3) + self.assertEqual(mock_check_call.call_count, 2) diff --git a/vars/runPerfTestScript.groovy b/vars/runPerfTestScript.groovy index 1a4fc500c5..17b3ca9044 100644 --- a/vars/runPerfTestScript.groovy +++ b/vars/runPerfTestScript.groovy @@ -4,7 +4,6 @@ void call(Map args = [:]) { def buildManifest = lib.jenkins.BuildManifest.new(readYaml(file: args.bundleManifest)) String artifactRootUrl = buildManifest.getArtifactRootUrl(jobName, args.buildId) - install_npm() install_dependencies() install_opensearch_infra_dependencies() withAWS(role: 'opensearch-test', roleAccount: "${AWS_ACCOUNT_PUBLIC}", duration: 900, roleSessionName: 'jenkins-session') { @@ -18,10 +17,15 @@ void call(Map args = [:]) { "--stack test-single-${args.buildId}", "--bundle-manifest ${args.bundleManifest}", "--config config.yml", - args.security ? "--security" : "" + args.security ? "--security" : "", + isNullOrEmpty(args.workload) ? "" : "--workload ${args.workload}", + isNullOrEmpty(args.testIterations) ? "" : "--test-iters ${args.testIterations}", + isNullOrEmpty(args.warmupIterations) ? "" : "--warmup-iters ${args.warmupIterations}", ].join(' ')) } +boolean isNullOrEmpty(String str) { return (str == null || str.allWhitespace) } + void install_opensearch_infra_dependencies() { sh''' pipenv install "dataclasses_json~=0.5" "aws_requests_auth~=0.4" "json2html~=1.3.0" @@ -30,18 +34,12 @@ void install_opensearch_infra_dependencies() { ''' } -void install_npm(){ - sh''' - sudo yum install -y gcc-c++ make - curl -sL https://rpm.nodesource.com/setup_16.x | sudo -E bash - - sudo yum install -y nodejs --enablerepo=nodesource - node -v - ''' -} - void install_dependencies() { sh ''' - sudo npm install -g aws-cdk - sudo npm install -g cdk-assume-role-credential-plugin + npm install -g fs-extra + npm install -g chalk@4.1.2 + npm install -g @aws-cdk/cloudformation-diff + npm install -g aws-cdk + npm install -g cdk-assume-role-credential-plugin@1.4.0 ''' } \ No newline at end of file From 5513bbf08d84621d1f680519f6845914adcfab95 Mon Sep 17 00:00:00 2001 From: Kunal Kotwani Date: Mon, 7 Mar 2022 21:52:17 -0800 Subject: [PATCH 03/12] Update parameter descriptions Signed-off-by: Kunal Kotwani --- jenkins/opensearch/perf-test.jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jenkins/opensearch/perf-test.jenkinsfile b/jenkins/opensearch/perf-test.jenkinsfile index 918b9a2579..0fb338bcbd 100644 --- a/jenkins/opensearch/perf-test.jenkinsfile +++ b/jenkins/opensearch/perf-test.jenkinsfile @@ -24,19 +24,19 @@ pipeline { string( defaultValue: 'nyc_taxis', name: 'TEST_WORKLOAD', - description: 'The agent label where the tests should be executed, e.g. Jenkins-Agent-al2-x64-c54xlarge-Docker-Host.', + description: 'The workload name from OpenSearch Benchmark Workloads for Mensor (internal client).', trim: true ) string( defaultValue: '1', name: 'TEST_ITERATIONS', - description: 'The agent label where the tests should be executed, e.g. Jenkins-Agent-al2-x64-c54xlarge-Docker-Host.', + description: 'Number of times to run a workload for Mensor (internal client).', trim: true ) string( defaultValue: '0', name: 'WARMUP_ITERATIONS', - description: 'The agent label where the tests should be executed, e.g. Jenkins-Agent-al2-x64-c54xlarge-Docker-Host.', + description: 'Number of times to run a workload before collecting data for Mensor (internal client).', trim: true ) } From 4f06122ac98447d96cc5f2cf9f4cb9475e20bc28 Mon Sep 17 00:00:00 2001 From: Kunal Kotwani Date: Fri, 18 Mar 2022 00:42:10 -0700 Subject: [PATCH 04/12] Add parallel stages for performance tests, add tests Signed-off-by: Kunal Kotwani --- jenkins/opensearch/perf-test.jenkinsfile | 170 ++++++++++++------ src/run_perf_test.py | 15 +- .../perf_test/perf_test_suite.py | 2 + tests/data/perf-test-config.yml | 9 + tests/jenkins/TestRunPerfTestScript.groovy | 1 - .../jenkins/data/opensearch-1.3.0-bundle.yml | 5 + .../jobs/RunPerfTestScript_Jenkinsfile | 3 +- .../jobs/RunPerfTestScript_Jenkinsfile.txt | 5 +- .../RunPerfTestScriptLibTest.groovy | 19 +- tests/test_run_perf_test.py | 92 ++++++++++ vars/runPerfTestScript.groovy | 8 +- 11 files changed, 249 insertions(+), 80 deletions(-) create mode 100644 tests/data/perf-test-config.yml create mode 100644 tests/test_run_perf_test.py diff --git a/jenkins/opensearch/perf-test.jenkinsfile b/jenkins/opensearch/perf-test.jenkinsfile index 0fb338bcbd..89ea71ee90 100644 --- a/jenkins/opensearch/perf-test.jenkinsfile +++ b/jenkins/opensearch/perf-test.jenkinsfile @@ -2,11 +2,11 @@ lib = library(identifier: "jenkins@20211118", retriever: legacySCM(scm)) pipeline { agent none - options { + options { timeout(time: 10, unit: 'HOURS') } environment { - AGENT_X64 = 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' + AGENT_LABEL = 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' AGENT_IMAGE = 'opensearchstaging/ci-runner:ci-runner-centos7-v1' BUNDLE_MANIFEST = "bundle-manifest.yml" } @@ -53,66 +53,130 @@ pipeline { currentBuild.result = 'ABORTED' error("Performance Tests failed to start. Missing parameter: GITHUB_TOKEN.") } + if (TEST_ITERATIONS != null && !TEST_ITERATIONS.isInteger()) { + currentBuild.result = 'ABORTED' + error("Performance Tests failed to start. Invalid value for parameter: TEST_ITERATIONS. Value should be an integer.") + } + if (WARMUP_ITERATIONS != null && !WARMUP_ITERATIONS.isInteger()) { + currentBuild.result = 'ABORTED' + error("Performance Tests failed to start. Invalid value for parameter: WARMUP_ITERATIONS. Value should be an integer.") + } } } } stage('perf-test') { - agent { - docker { - label AGENT_X64 - image AGENT_IMAGE - alwaysPull true - } - } - steps { - script { - def bundleManifestObj = downloadBuildManifest( - url: BUNDLE_MANIFEST_URL, - path: BUNDLE_MANIFEST - ) - String buildId = bundleManifestObj.getArtifactBuildId() - env.BUILD_ID = buildId - env.HAS_SECURITY = bundleManifestObj.components.containsKey("security") - echo "BUNDLE_MANIFEST: ${BUNDLE_MANIFEST}" - echo "BUILD_ID: ${BUILD_ID}" - echo "Security: ${HAS_SECURITY}" - - lib.jenkins.Messages.new(this).add('perf-test', "[Test]Executing performance tests for #${BUILD_ID}") + parallel { + stage('test-with-security') { + agent { + docker { + label AGENT_LABEL + image AGENT_IMAGE + alwaysPull true + } + } + steps { + script { - runPerfTestScript(bundleManifest: "${BUNDLE_MANIFEST}", - buildId: "${BUILD_ID}", - security: false, - workload: TEST_WORKLOAD, - testIterations: TEST_ITERATIONS, - warmupIterations: WARMUP_ITERATIONS) + def bundleManifestObj = downloadBuildManifest( + url: BUNDLE_MANIFEST_URL, + path: BUNDLE_MANIFEST + ) + String buildId = bundleManifestObj.getArtifactBuildId() + env.BUILD_ID = buildId + env.HAS_SECURITY = bundleManifestObj.components.containsKey("security") + env.ARCHITECTURE = bundleManifestObj.getArtifactArchitecture() + echo "BUNDLE_MANIFEST: ${BUNDLE_MANIFEST}" + echo "BUILD_ID: ${BUILD_ID}" + echo "BUILD_ID: ${BUILD_ID}" + echo "Security: ${HAS_SECURITY}" + echo "Architecture: ${ARCHITECTURE}" - lib.jenkins.Messages.new(this).add('perf-test', "[Test]Performance tests without security for ${BUILD_ID} completed") + lib.jenkins.Messages.new(this).add('perf-test', "Performance tests for #${BUILD_ID}") + if (env.HAS_SECURITY) { + runPerfTestScript(bundleManifest: "${BUNDLE_MANIFEST}", + buildId: "${BUILD_ID}", + architecture: "${ARCHITECTURE}", + insecure: false, + workload: TEST_WORKLOAD, + testIterations: TEST_ITERATIONS, + warmupIterations: WARMUP_ITERATIONS) - if(env.HAS_SECURITY) { - runPerfTestScript(bundleManifest: "${BUNDLE_MANIFEST}", - buildId: "${BUILD_ID}", - security: true, - workload: TEST_WORKLOAD, - testIterations: TEST_ITERATIONS, - warmupIterations: WARMUP_ITERATIONS) - lib.jenkins.Messages.new(this).add('perf-test', "[Test]Performance tests with security for ${BUILD_ID} completed") + lib.jenkins.Messages.new(this).add('perf-test', "Performance tests with security for ${BUILD_ID} completed") + } + } + } + post { + success { + script { + uploadTestResults( + buildManifestFileName: "${BUNDLE_MANIFEST}", + jobName: 'perf-test', + buildNumber: "${BUILD_ID}" + ) + } + postCleanup() + } + failure { + postCleanup() + } + aborted { + postCleanup() + } } - } - } + stage('test-without-security') { + agent { + docker { + label AGENT_LABEL + image AGENT_IMAGE + alwaysPull true + } + } + steps { + script { + def bundleManifestObj = downloadBuildManifest( + url: BUNDLE_MANIFEST_URL, + path: BUNDLE_MANIFEST + ) + String buildId = bundleManifestObj.getArtifactBuildId() + env.BUILD_ID = buildId + env.ARCHITECTURE = bundleManifestObj.getArtifactArchitecture() + echo "BUNDLE_MANIFEST: ${BUNDLE_MANIFEST}" + echo "BUILD_ID: ${BUILD_ID}" + echo "BUILD_ID: ${BUILD_ID}" + echo "Architecture: ${ARCHITECTURE}" + + lib.jenkins.Messages.new(this).add('perf-test', "Performance tests for #${BUILD_ID}") - post { - success { - script { - uploadTestResults( - buildManifestFileName: "${BUNDLE_MANIFEST}", - jobName: 'perf-test', - buildNumber: "${BUILD_ID}" - ) + runPerfTestScript(bundleManifest: "${BUNDLE_MANIFEST}", + buildId: "${BUILD_ID}", + architecture: "${ARCHITECTURE}", + insecure: true, + workload: TEST_WORKLOAD, + testIterations: TEST_ITERATIONS, + warmupIterations: WARMUP_ITERATIONS) + + lib.jenkins.Messages.new(this).add('perf-test', "Performance tests without security for ${BUILD_ID} completed") + } + } + post { + success { + script { + uploadTestResults( + buildManifestFileName: "${BUNDLE_MANIFEST}", + jobName: 'perf-test', + buildNumber: "${BUILD_ID}" + ) + } + postCleanup() + } + failure { + postCleanup() + } + aborted { + postCleanup() + } } - } - always { - postCleanup() } } } @@ -136,7 +200,7 @@ pipeline { } failure { node(AGENT_X64) { - script { + script { def stashed = lib.jenkins.Messages.new(this).get(['perf-test']) publishNotification( icon: ':warning:', @@ -150,4 +214,4 @@ pipeline { } } } -} +} \ No newline at end of file diff --git a/src/run_perf_test.py b/src/run_perf_test.py index 8907803632..caca751578 100644 --- a/src/run_perf_test.py +++ b/src/run_perf_test.py @@ -36,8 +36,8 @@ def main(): parser.add_argument("--bundle-manifest", type=argparse.FileType("r"), help="Bundle Manifest file.", required=True) parser.add_argument("--stack", dest="stack", help="Stack name for performance test") parser.add_argument("--config", type=argparse.FileType("r"), help="Config file.", required=True) - parser.add_argument("--security", dest="security", action="store_true", - help="Security of the cluster should be True/False", + parser.add_argument("--force-insecure-mode", dest="insecure", action="store_true", + help="Force the security of the cluster to be disabled.", default=False) parser.add_argument("--keep", dest="keep", action="store_true", help="Do not delete the working temporary directory.") parser.add_argument("--workload", default="nyc_taxis", help="Mensor (internal client) param - Workload name from OpenSeach Benchmark Workloads") @@ -49,15 +49,20 @@ def main(): manifest = BundleManifest.from_file(args.bundle_manifest) config = yaml.safe_load(args.config) - tests_dir = os.path.join(os.getcwd(), "test-results", "perf-test") + security = "security" in manifest.components and not args.insecure + if security: + tests_dir = os.path.join(os.getcwd(), "test-results", "perf-test", "with-security") + else: + tests_dir = os.path.join(os.getcwd(), "test-results", "perf-test", "without-security") os.makedirs(tests_dir, exist_ok=True) with TemporaryDirectory(keep=args.keep, chdir=True) as work_dir: current_workspace = os.path.join(work_dir.name, "infra") with GitRepository(get_infra_repo_url(), "main", current_workspace): with WorkingDirectory(current_workspace): - with PerfTestCluster.create(manifest, config, args.stack, args.security, current_workspace) as (test_cluster_endpoint, test_cluster_port): - perf_test_suite = PerfTestSuite(manifest, test_cluster_endpoint, args.security, + with PerfTestCluster.create(manifest, config, args.stack, security, current_workspace) \ + as (test_cluster_endpoint, test_cluster_port): + perf_test_suite = PerfTestSuite(manifest, test_cluster_endpoint, security, current_workspace, tests_dir, args) retry_call(perf_test_suite.execute, tries=3, delay=60, backoff=2) diff --git a/src/test_workflow/perf_test/perf_test_suite.py b/src/test_workflow/perf_test/perf_test_suite.py index b551d509d5..3f6fc55ce8 100644 --- a/src/test_workflow/perf_test/perf_test_suite.py +++ b/src/test_workflow/perf_test/perf_test_suite.py @@ -26,6 +26,8 @@ def __init__(self, bundle_manifest, endpoint, security, current_workspace, test_ self.command = ( f"pipenv run python test_config.py -i {self.endpoint} -b {self.manifest.build.id}" f" -a {self.manifest.build.architecture} -p {test_results_path}" + f" --workload {self.args.workload} --workload-options '{self.args.workload_options}'" + f" --warmup-iters {self.args.warmup_iters} --test-iters {self.args.test_iters}" ) def execute(self): diff --git a/tests/data/perf-test-config.yml b/tests/data/perf-test-config.yml new file mode 100644 index 0000000000..1625ca748e --- /dev/null +++ b/tests/data/perf-test-config.yml @@ -0,0 +1,9 @@ +--- +Description: Configuration file to store contants required to run rally for performance test +Constants: + Repository: https://github.com/opensearch-project/ + SecurityGroupId: test-security + VpcId: test-vpc + AccountId: 123456 + Region: eu-east-1 + Role: test-role-name diff --git a/tests/jenkins/TestRunPerfTestScript.groovy b/tests/jenkins/TestRunPerfTestScript.groovy index 1fbfa10a89..76204281c9 100644 --- a/tests/jenkins/TestRunPerfTestScript.groovy +++ b/tests/jenkins/TestRunPerfTestScript.groovy @@ -16,7 +16,6 @@ class TestRunPerfTestScript extends BuildPipelineTest { @Before void setUp() { this.registerLibTester(new RunPerfTestScriptLibTester( - 'dummy_job', 'tests/jenkins/data/opensearch-1.3.0-bundle.yml', '1236', 'false', diff --git a/tests/jenkins/data/opensearch-1.3.0-bundle.yml b/tests/jenkins/data/opensearch-1.3.0-bundle.yml index 8a70c63e74..d5ee76eaee 100644 --- a/tests/jenkins/data/opensearch-1.3.0-bundle.yml +++ b/tests/jenkins/data/opensearch-1.3.0-bundle.yml @@ -14,3 +14,8 @@ components: ref: 1.x commit_id: 22408088f002a4fc8cdd3b2ed7438866c14c5069 location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/1236/linux/x64/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64.tar.gz + - name: security + repository: https://github.com/opensearch-project/security.git + ref: 1.x + commit_id: 9bf0ab861eae018352427e44b434aa12c1577fa5 + location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/1236/linux/x64/builds/opensearch/plugins/opensearch-security-1.3.0.0.zip diff --git a/tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile b/tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile index a3261db69b..65f45b6b55 100644 --- a/tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile +++ b/tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile @@ -13,10 +13,9 @@ pipeline { steps { script { runPerfTestScript( - jobName: 'dummy_job', bundleManifest: 'tests/jenkins/data/opensearch-1.3.0-bundle.yml', buildId: '1236', - security: 'false', + insecure: 'false', workload: 'nyc_taxis', testIterations: '1', warmupIterations: '1' diff --git a/tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile.txt b/tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile.txt index 2b58cb26e5..12762e3bf1 100644 --- a/tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile.txt +++ b/tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile.txt @@ -3,12 +3,11 @@ RunPerfTestScript_Jenkinsfile.echo(Executing on agent [label:none]) RunPerfTestScript_Jenkinsfile.stage(perf-test, groovy.lang.Closure) RunPerfTestScript_Jenkinsfile.script(groovy.lang.Closure) - RunPerfTestScript_Jenkinsfile.runPerfTestScript({jobName=dummy_job, bundleManifest=tests/jenkins/data/opensearch-1.3.0-bundle.yml, buildId=1236, security=false, workload=nyc_taxis, testIterations=1, warmupIterations=1}) + RunPerfTestScript_Jenkinsfile.runPerfTestScript({bundleManifest=tests/jenkins/data/opensearch-1.3.0-bundle.yml, buildId=1236, insecure=false, workload=nyc_taxis, testIterations=1, warmupIterations=1}) runPerfTestScript.legacySCM(groovy.lang.Closure) runPerfTestScript.library({identifier=jenkins@20211123, retriever=null}) runPerfTestScript.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) BuildManifest.asBoolean() - BuildManifest.getArtifactRootUrl(dummy_job, 1236) runPerfTestScript.sh( npm install -g fs-extra npm install -g chalk@4.1.2 @@ -23,4 +22,4 @@ ) runPerfTestScript.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) runPerfTestScript.s3Download({file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true}) - runPerfTestScript.sh(./test.sh perf-test --stack test-single-security-1236 --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config config.yml --security --workload nyc_taxis --test-iters 1 --warmup-iters 1) + runPerfTestScript.sh(./test.sh perf-test --stack test-single-1236-null --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config config.yml --force-insecure-mode --workload nyc_taxis --test-iters 1 --warmup-iters 1) diff --git a/tests/jenkins/lib-testers/RunPerfTestScriptLibTest.groovy b/tests/jenkins/lib-testers/RunPerfTestScriptLibTest.groovy index 6eed917a93..912b7e41ca 100644 --- a/tests/jenkins/lib-testers/RunPerfTestScriptLibTest.groovy +++ b/tests/jenkins/lib-testers/RunPerfTestScriptLibTest.groovy @@ -4,19 +4,17 @@ import static org.hamcrest.MatcherAssert.assertThat class RunPerfTestScriptLibTester extends LibFunctionTester { - private String jobName private String bundleManifest private String buildId - private String security + private String insecure private String workload private String testIterations private String warmupIterations - public RunPerfTestScriptLibTester(jobName, bundleManifest, buildId, security, workload, testIterations, warmupIterations){ - this.jobName = jobName + public RunPerfTestScriptLibTester(bundleManifest, buildId, insecure, workload, testIterations, warmupIterations){ this.bundleManifest = bundleManifest this.buildId = buildId - this.security = security + this.insecure = insecure this.workload = workload this.testIterations = testIterations this.warmupIterations = warmupIterations @@ -40,20 +38,19 @@ class RunPerfTestScriptLibTester extends LibFunctionTester { void parameterInvariantsAssertions(call) { assertThat(call.args.bundleManifest.first(), notNullValue()) assertThat(call.args.buildId.first(), notNullValue()) - assertThat(call.args.security.first(), notNullValue()) + assertThat(call.args.insecure.first(), notNullValue()) assertThat(call.args.workload.first(), notNullValue()) assertThat(call.args.testIterations.first(), notNullValue()) assertThat(call.args.warmupIterations.first(), notNullValue()) } boolean expectedParametersMatcher(call) { - return call.args.jobName.first().toString().equals(this.jobName) - && call.args.bundleManifest.first().toString().equals(this.bundleManifest) + return call.args.bundleManifest.first().toString().equals(this.bundleManifest) && call.args.buildId.first().toString().equals(this.buildId) - && call.args.security.first().toString().equals(this.security) + && call.args.insecure.first().toString().equals(this.insecure) && call.args.workload.first().toString().equals(this.workload) - && call.args.testIterations.first().toString().equals(this.testIterations) - && call.args.warmupIterations.first().toString().equals(this.warmupIterations) + && call.args.testIterations.first().toInteger().equals(this.testIterations.toInteger()) + && call.args.warmupIterations.first().toInteger().equals(this.warmupIterations.toInteger()) } String libFunctionName() { diff --git a/tests/test_run_perf_test.py b/tests/test_run_perf_test.py new file mode 100644 index 0000000000..6f80d36ee0 --- /dev/null +++ b/tests/test_run_perf_test.py @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +import os +import tempfile +import unittest +from unittest.mock import Mock, patch + +import pytest + +from run_perf_test import main + + +class TestRunPerfTest(unittest.TestCase): + @pytest.fixture(autouse=True) + def capfd(self, capfd): + self.capfd = capfd + + @patch("argparse._sys.argv", ["run_perf_test.py", "--help"]) + def test_usage(self): + with self.assertRaises(SystemExit): + main() + + out, _ = self.capfd.readouterr() + self.assertTrue(out.startswith("usage:")) + + BUNDLE_MANIFEST_PATH = os.path.join( + os.path.dirname(__file__), + "jenkins", + "data", + ) + + CONFIG_ROOT_PATH = os.path.join( + os.path.dirname(__file__), + "data", + ) + + OPENSEARCH_BUNDLE_MANIFEST = os.path.realpath(os.path.join(BUNDLE_MANIFEST_PATH, "opensearch-1.3.0-bundle.yml")) + PERF_TEST_CONFIG = os.path.realpath(os.path.join(CONFIG_ROOT_PATH, "perf-test-config.yml")) + + @patch("argparse._sys.argv", ["run_perf_test.py", "--bundle-manifest", OPENSEARCH_BUNDLE_MANIFEST, + "--stack", "test-stack", "--config", PERF_TEST_CONFIG]) + @patch("run_perf_test.PerfTestCluster.create") + @patch("run_perf_test.PerfTestSuite") + @patch("run_perf_test.WorkingDirectory") + @patch("run_perf_test.TemporaryDirectory") + @patch("run_perf_test.GitRepository") + def test_default_execute_perf_test(self, mock_git, mock_temp, mock_working_dir, mock_suite, mock_cluster, *mocks): + mock_temp.return_value.__enter__.return_value.name = tempfile.gettempdir() + mock_create = Mock() + mock_create.__enter__ = Mock(return_value=('test-endpoint', 1234)) + mock_create.__exit__ = Mock(return_value=None) + mock_cluster.return_value = mock_create + + mock_execute = Mock() + mock_suite.return_value.execute = mock_execute + + main() + self.assertEqual(1, mock_cluster.call_count) + self.assertEqual(1, mock_suite.call_count) + self.assertEqual(1, mock_git.call_count) + self.assertEqual(1, mock_execute.call_count) + self.assertEqual([], mock_execute.call_args) + self.assertIn(True, mock_suite.call_args.args) + + @patch("argparse._sys.argv", ["run_perf_test.py", "--bundle-manifest", OPENSEARCH_BUNDLE_MANIFEST, + "--stack", "test-stack", "--config", PERF_TEST_CONFIG, "--force-insecure-mode"]) + @patch("run_perf_test.PerfTestCluster.create") + @patch("run_perf_test.PerfTestSuite") + @patch("run_perf_test.WorkingDirectory") + @patch("run_perf_test.TemporaryDirectory") + @patch("run_perf_test.GitRepository") + def test_with_security_execute_perf_test(self, mock_git, mock_temp, mock_working_dir, mock_suite, mock_cluster, *mocks): + mock_temp.return_value.__enter__.return_value.name = tempfile.gettempdir() + mock_create = Mock() + mock_create.__enter__ = Mock(return_value=('test-endpoint', 1234)) + mock_create.__exit__ = Mock(return_value=None) + mock_cluster.return_value = mock_create + + mock_execute = Mock() + mock_suite.return_value.execute = mock_execute + + main() + self.assertEqual(1, mock_cluster.call_count) + self.assertEqual(1, mock_suite.call_count) + self.assertEqual(1, mock_git.call_count) + self.assertEqual(1, mock_execute.call_count) + self.assertEqual([], mock_execute.call_args) + self.assertIn(False, mock_suite.call_args.args) diff --git a/vars/runPerfTestScript.groovy b/vars/runPerfTestScript.groovy index 17b3ca9044..c90f03296e 100644 --- a/vars/runPerfTestScript.groovy +++ b/vars/runPerfTestScript.groovy @@ -1,8 +1,6 @@ void call(Map args = [:]) { - String jobName = args.jobName ?: 'distribution-build-opensearch' lib = library(identifier: 'jenkins@20211123', retriever: legacySCM(scm)) def buildManifest = lib.jenkins.BuildManifest.new(readYaml(file: args.bundleManifest)) - String artifactRootUrl = buildManifest.getArtifactRootUrl(jobName, args.buildId) install_dependencies() install_opensearch_infra_dependencies() @@ -13,11 +11,11 @@ void call(Map args = [:]) { sh([ './test.sh', 'perf-test', - args.security ? "--stack test-single-security-${args.buildId}" : - "--stack test-single-${args.buildId}", + args.insecure ? "--stack test-single-${args.buildId}-${args.architecture}" : + "--stack test-single-security-${args.buildId}-${args.architecture}", "--bundle-manifest ${args.bundleManifest}", "--config config.yml", - args.security ? "--security" : "", + args.insecure ? "--force-insecure-mode" : "", isNullOrEmpty(args.workload) ? "" : "--workload ${args.workload}", isNullOrEmpty(args.testIterations) ? "" : "--test-iters ${args.testIterations}", isNullOrEmpty(args.warmupIterations) ? "" : "--warmup-iters ${args.warmupIterations}", From 57c89779bf3215656006a4d09407d209865e8adb Mon Sep 17 00:00:00 2001 From: Kunal Kotwani Date: Fri, 18 Mar 2022 12:42:10 -0700 Subject: [PATCH 05/12] Update tests as per Python3.7 Signed-off-by: Kunal Kotwani --- Pipfile.lock | 133 ++++++++++++++++++++++++------------ tests/test_run_perf_test.py | 4 +- 2 files changed, 93 insertions(+), 44 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 7835536fec..f5e633f5ee 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -149,11 +149,11 @@ }, "identify": { "hashes": [ - "sha256:2986942d3974c8f2e5019a190523b0b0e2a07cb8e89bf236727fb4b26f27f8fd", - "sha256:fd906823ed1db23c7a48f9b176a1d71cb8abede1e21ebe614bac7bdd688d9213" + "sha256:3f3244a559290e7d3deb9e9adc7b33594c1bc85a9dd82e0f1be519bf12a1ec17", + "sha256:5f06b14366bd1facb88b00540a1de05b69b310cbc2654db3c7e07fa3a4339323" ], "markers": "python_version >= '3.7'", - "version": "==2.4.11" + "version": "==2.4.12" }, "idna": { "hashes": [ @@ -163,6 +163,14 @@ "markers": "python_version >= '3'", "version": "==3.3" }, + "importlib-metadata": { + "hashes": [ + "sha256:1208431ca90a8cca1a6b8af391bb53c1a2db74e5d1cef6ddced95d4b2062edc6", + "sha256:ea4c597ebf37142f827b8f39299579e31685c31d3a438b59f469406afd0f2539" + ], + "markers": "python_version < '3.8'", + "version": "==4.11.3" + }, "iniconfig": { "hashes": [ "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3", @@ -195,29 +203,32 @@ }, "mypy": { "hashes": [ - "sha256:0038b21890867793581e4cb0d810829f5fd4441aa75796b53033af3aa30430ce", - "sha256:1171f2e0859cfff2d366da2c7092b06130f232c636a3f7301e3feb8b41f6377d", - "sha256:1b06268df7eb53a8feea99cbfff77a6e2b205e70bf31743e786678ef87ee8069", - "sha256:1b65714dc296a7991000b6ee59a35b3f550e0073411ac9d3202f6516621ba66c", - "sha256:1bf752559797c897cdd2c65f7b60c2b6969ffe458417b8d947b8340cc9cec08d", - "sha256:300717a07ad09525401a508ef5d105e6b56646f7942eb92715a1c8d610149714", - "sha256:3c5b42d0815e15518b1f0990cff7a705805961613e701db60387e6fb663fe78a", - "sha256:4365c60266b95a3f216a3047f1d8e3f895da6c7402e9e1ddfab96393122cc58d", - "sha256:50c7346a46dc76a4ed88f3277d4959de8a2bd0a0fa47fa87a4cde36fe247ac05", - "sha256:5b56154f8c09427bae082b32275a21f500b24d93c88d69a5e82f3978018a0266", - "sha256:74f7eccbfd436abe9c352ad9fb65872cc0f1f0a868e9d9c44db0893440f0c697", - "sha256:7b3f6f557ba4afc7f2ce6d3215d5db279bcf120b3cfd0add20a5d4f4abdae5bc", - "sha256:8c11003aaeaf7cc2d0f1bc101c1cc9454ec4cc9cb825aef3cafff8a5fdf4c799", - "sha256:8ca7f8c4b1584d63c9a0f827c37ba7a47226c19a23a753d52e5b5eddb201afcd", - "sha256:c89702cac5b302f0c5d33b172d2b55b5df2bede3344a2fbed99ff96bddb2cf00", - "sha256:d8f1ff62f7a879c9fe5917b3f9eb93a79b78aad47b533911b853a757223f72e7", - "sha256:d9d2b84b2007cea426e327d2483238f040c49405a6bf4074f605f0156c91a47a", - "sha256:e839191b8da5b4e5d805f940537efcaa13ea5dd98418f06dc585d2891d228cf0", - "sha256:f9fe20d0872b26c4bba1c1be02c5340de1019530302cf2dcc85c7f9fc3252ae0", - "sha256:ff3bf387c14c805ab1388185dd22d6b210824e164d4bb324b195ff34e322d166" + "sha256:080097eee5393fd740f32c63f9343580aaa0fb1cda0128fd859dfcf081321c3d", + "sha256:0d3bcbe146247997e03bf030122000998b076b3ac6925b0b6563f46d1ce39b50", + "sha256:0dd441fbacf48e19dc0c5c42fafa72b8e1a0ba0a39309c1af9c84b9397d9b15a", + "sha256:108f3c7e14a038cf097d2444fa0155462362c6316e3ecb2d70f6dd99cd36084d", + "sha256:3bada0cf7b6965627954b3a128903a87cac79a79ccd83b6104912e723ef16c7b", + "sha256:3cf77f138efb31727ee7197bc824c9d6d7039204ed96756cc0f9ca7d8e8fc2a4", + "sha256:42c216a33d2bdba08098acaf5bae65b0c8196afeb535ef4b870919a788a27259", + "sha256:465a6ce9ca6268cadfbc27a2a94ddf0412568a6b27640ced229270be4f5d394d", + "sha256:6a8e1f63357851444940351e98fb3252956a15f2cabe3d698316d7a2d1f1f896", + "sha256:745071762f32f65e77de6df699366d707fad6c132a660d1342077cbf671ef589", + "sha256:818cfc51c25a5dbfd0705f3ac1919fff6971eb0c02e6f1a1f6a017a42405a7c0", + "sha256:8e5974583a77d630a5868eee18f85ac3093caf76e018c510aeb802b9973304ce", + "sha256:8eaf55fdf99242a1c8c792247c455565447353914023878beadb79600aac4a2a", + "sha256:98f61aad0bb54f797b17da5b82f419e6ce214de0aa7e92211ebee9e40eb04276", + "sha256:b2ce2788df0c066c2ff4ba7190fa84f18937527c477247e926abeb9b1168b8cc", + "sha256:b30d29251dff4c59b2e5a1fa1bab91ff3e117b4658cb90f76d97702b7a2ae699", + "sha256:bf446223b2e0e4f0a4792938e8d885e8a896834aded5f51be5c3c69566495540", + "sha256:cbcc691d8b507d54cb2b8521f0a2a3d4daa477f62fe77f0abba41e5febb377b7", + "sha256:d051ce0946521eba48e19b25f27f98e5ce4dbc91fff296de76240c46b4464df0", + "sha256:d61b73c01fc1de799226963f2639af831307fe1556b04b7c25e2b6c267a3bc76", + "sha256:eea10982b798ff0ccc3b9e7e42628f932f552c5845066970e67cd6858655d52c", + "sha256:f79137d012ff3227866222049af534f25354c07a0d6b9a171dba9f1d6a1fdef4", + "sha256:fc5ecff5a3bbfbe20091b1cad82815507f5ae9c380a3a9bf40f740c70ce30a9b" ], "index": "pypi", - "version": "==0.931" + "version": "==0.941" }, "mypy-extensions": { "hashes": [ @@ -344,11 +355,11 @@ }, "pytest": { "hashes": [ - "sha256:9ce3ff477af913ecf6321fe337b93a2c0dcf2a0a1439c43f5452112c1e4280db", - "sha256:e30905a0c131d3d94b89624a1cc5afec3e0ba2fbdb151867d8e0ebd49850f171" + "sha256:841132caef6b1ad17a9afde46dc4f6cfa59a05f9555aae5151f73bdf2820ca63", + "sha256:92f723789a8fdd7180b6b06483874feca4c48a5c76968e03bb3e7f806a1869ea" ], "index": "pypi", - "version": "==7.0.1" + "version": "==7.1.1" }, "pytest-cov": { "hashes": [ @@ -419,11 +430,11 @@ }, "setuptools": { "hashes": [ - "sha256:2347b2b432c891a863acadca2da9ac101eae6169b1d3dfee2ec605ecd50dbfe5", - "sha256:e4f30b9f84e5ab3decf945113119649fec09c1fc3507c6ebffec75646c56e62b" + "sha256:6599055eeb23bfef457d5605d33a4d68804266e6cb430b0fb12417c5efeae36c", + "sha256:782ef48d58982ddb49920c11a0c5c9c0b02e7d7d1c2ad0aa44e1a1e133051c96" ], "markers": "python_version >= '3.7'", - "version": "==60.9.3" + "version": "==60.10.0" }, "six": { "hashes": [ @@ -457,6 +468,36 @@ "markers": "python_version >= '3.7'", "version": "==2.0.1" }, + "typed-ast": { + "hashes": [ + "sha256:0eb77764ea470f14fcbb89d51bc6bbf5e7623446ac4ed06cbd9ca9495b62e36e", + "sha256:1098df9a0592dd4c8c0ccfc2e98931278a6c6c53cb3a3e2cf7e9ee3b06153344", + "sha256:183b183b7771a508395d2cbffd6db67d6ad52958a5fdc99f450d954003900266", + "sha256:18fe320f354d6f9ad3147859b6e16649a0781425268c4dde596093177660e71a", + "sha256:26a432dc219c6b6f38be20a958cbe1abffcc5492821d7e27f08606ef99e0dffd", + "sha256:294a6903a4d087db805a7656989f613371915fc45c8cc0ddc5c5a0a8ad9bea4d", + "sha256:31d8c6b2df19a777bc8826770b872a45a1f30cfefcfd729491baa5237faae837", + "sha256:33b4a19ddc9fc551ebabca9765d54d04600c4a50eda13893dadf67ed81d9a098", + "sha256:42c47c3b43fe3a39ddf8de1d40dbbfca60ac8530a36c9b198ea5b9efac75c09e", + "sha256:525a2d4088e70a9f75b08b3f87a51acc9cde640e19cc523c7e41aa355564ae27", + "sha256:58ae097a325e9bb7a684572d20eb3e1809802c5c9ec7108e85da1eb6c1a3331b", + "sha256:676d051b1da67a852c0447621fdd11c4e104827417bf216092ec3e286f7da596", + "sha256:74cac86cc586db8dfda0ce65d8bcd2bf17b58668dfcc3652762f3ef0e6677e76", + "sha256:8c08d6625bb258179b6e512f55ad20f9dfef019bbfbe3095247401e053a3ea30", + "sha256:90904d889ab8e81a956f2c0935a523cc4e077c7847a836abee832f868d5c26a4", + "sha256:963a0ccc9a4188524e6e6d39b12c9ca24cc2d45a71cfdd04a26d883c922b4b78", + "sha256:bbebc31bf11762b63bf61aaae232becb41c5bf6b3461b80a4df7e791fabb3aca", + "sha256:bc2542e83ac8399752bc16e0b35e038bdb659ba237f4222616b4e83fb9654985", + "sha256:c29dd9a3a9d259c9fa19d19738d021632d673f6ed9b35a739f48e5f807f264fb", + "sha256:c7407cfcad702f0b6c0e0f3e7ab876cd1d2c13b14ce770e412c0c4b9728a0f88", + "sha256:da0a98d458010bf4fe535f2d1e367a2e2060e105978873c04c04212fb20543f7", + "sha256:df05aa5b241e2e8045f5f4367a9f6187b09c4cdf8578bb219861c4e27c443db5", + "sha256:f290617f74a610849bd8f5514e34ae3d09eafd521dceaa6cf68b3f4414266d4e", + "sha256:f30ddd110634c2d7534b2d4e0e22967e88366b0d356b24de87419cc4410c41b7" + ], + "markers": "python_version < '3.8'", + "version": "==1.5.2" + }, "types-pyyaml": { "hashes": [ "sha256:3f4daa754357491625ae8c3a39c9e1b0d7cd5632bc4e1c35e7a7f75a64aa124b", @@ -467,18 +508,18 @@ }, "types-requests": { "hashes": [ - "sha256:506279bad570c7b4b19ac1f22e50146538befbe0c133b2cea66a9b04a533a859", - "sha256:6a7ed24b21780af4a5b5e24c310b2cd885fb612df5fd95584d03d87e5f2a195a" + "sha256:5d6f77f3c7565659bdb7b7bce1d33d1abb7d0b056138cac714860e13da2f19df", + "sha256:cf0646031dd6307113b37814f743c04f0707a3357378c2bb1326f848412f5ba9" ], "index": "pypi", - "version": "==2.27.11" + "version": "==2.27.13" }, "types-urllib3": { "hashes": [ - "sha256:a26898f530e6c3f43f25b907f2b884486868ffd56a9faa94cbf9b3eb6e165d6a", - "sha256:d755278d5ecd7a7a6479a190e54230f241f1a99c19b81518b756b19dc69e518c" + "sha256:24d64e441168851eb05f1d022de18ae31558f5649c8f1117e384c2e85e31315b", + "sha256:bd0abc01e9fb963e4fddd561a56d21cc371b988d1245662195c90379077139cd" ], - "version": "==1.26.10" + "version": "==1.26.11" }, "typing-extensions": { "hashes": [ @@ -490,11 +531,11 @@ }, "urllib3": { "hashes": [ - "sha256:000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed", - "sha256:0e7c33d9a63e7ddfcb86780aac87befc2fbddf46c58dbb487e0855f7ceec283c" + "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14", + "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", - "version": "==1.26.8" + "version": "==1.26.9" }, "validators": { "hashes": [ @@ -514,11 +555,11 @@ }, "yamlfix": { "hashes": [ - "sha256:3ea6ae93d68d1c4ecafaddd2a53870b6da0ff657eb34d7b78ff40ea485b15430", - "sha256:c8d5497b7b8733c21e215fd64510d96f5363bcedbdc1660ee56311a171d7fef5" + "sha256:ddcc14d6330c211d92a93d5662d3ef1cde61ab9b27d012f87f716a285425ccc8", + "sha256:ddf097c99fd572ccf6fb444227c0575308d16531cba7d231a689ccd6005e18a0" ], "index": "pypi", - "version": "==0.8.2" + "version": "==0.9.0" }, "yamllint": { "hashes": [ @@ -526,6 +567,14 @@ ], "index": "pypi", "version": "==1.26.3" + }, + "zipp": { + "hashes": [ + "sha256:9f50f446828eb9d45b267433fd3e9da8d801f614129124863f9c51ebceafb87d", + "sha256:b47250dd24f92b7dd6a0a8fc5244da14608f3ca90a5efcd37a3b1642fac9a375" + ], + "markers": "python_version >= '3.7'", + "version": "==3.7.0" } }, "develop": {} diff --git a/tests/test_run_perf_test.py b/tests/test_run_perf_test.py index 6f80d36ee0..6dcb216435 100644 --- a/tests/test_run_perf_test.py +++ b/tests/test_run_perf_test.py @@ -64,7 +64,7 @@ def test_default_execute_perf_test(self, mock_git, mock_temp, mock_working_dir, self.assertEqual(1, mock_git.call_count) self.assertEqual(1, mock_execute.call_count) self.assertEqual([], mock_execute.call_args) - self.assertIn(True, mock_suite.call_args.args) + self.assertIn(True, mock_suite.call_args[0]) @patch("argparse._sys.argv", ["run_perf_test.py", "--bundle-manifest", OPENSEARCH_BUNDLE_MANIFEST, "--stack", "test-stack", "--config", PERF_TEST_CONFIG, "--force-insecure-mode"]) @@ -89,4 +89,4 @@ def test_with_security_execute_perf_test(self, mock_git, mock_temp, mock_working self.assertEqual(1, mock_git.call_count) self.assertEqual(1, mock_execute.call_count) self.assertEqual([], mock_execute.call_args) - self.assertIn(False, mock_suite.call_args.args) + self.assertIn(False, mock_suite.call_args[0]) From eee81d011a34ad3a9b76a58f1b7cf38504b97eb1 Mon Sep 17 00:00:00 2001 From: Kunal Kotwani Date: Tue, 22 Mar 2022 10:18:32 -0700 Subject: [PATCH 06/12] Add conditional stage for performance tests Signed-off-by: Kunal Kotwani --- jenkins/opensearch/perf-test.jenkinsfile | 48 +++++++++++------- tests/jenkins/TestRunPerfTestScript.groovy | 8 ++- .../opensearch-1.3.0-non-security-bundle.yml | 16 ++++++ ...nPerfTestScriptWithoutSecurity_Jenkinsfile | 27 ++++++++++ ...fTestScriptWithoutSecurity_Jenkinsfile.txt | 50 +++++++++++++++++++ .../RunPerfTestScriptLibTest.groovy | 4 ++ 6 files changed, 132 insertions(+), 21 deletions(-) create mode 100644 tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml create mode 100644 tests/jenkins/jobs/RunPerfTestScriptWithoutSecurity_Jenkinsfile create mode 100644 tests/jenkins/jobs/RunPerfTestScriptWithoutSecurity_Jenkinsfile.txt diff --git a/jenkins/opensearch/perf-test.jenkinsfile b/jenkins/opensearch/perf-test.jenkinsfile index 89ea71ee90..a3e2a95663 100644 --- a/jenkins/opensearch/perf-test.jenkinsfile +++ b/jenkins/opensearch/perf-test.jenkinsfile @@ -42,7 +42,14 @@ pipeline { } stages { - stage('validate-parameters') { + stage('validate-and-set-parameters') { + agent { + docker { + label AGENT_LABEL + image AGENT_IMAGE + alwaysPull true + } + } steps { script { if (BUNDLE_MANIFEST_URL == '') { @@ -61,6 +68,14 @@ pipeline { currentBuild.result = 'ABORTED' error("Performance Tests failed to start. Invalid value for parameter: WARMUP_ITERATIONS. Value should be an integer.") } + def bundleManifestObj = downloadBuildManifest( + url: BUNDLE_MANIFEST_URL, + path: BUNDLE_MANIFEST + ) + String buildId = bundleManifestObj.getArtifactBuildId() + env.BUILD_ID = buildId + env.HAS_SECURITY = bundleManifestObj.components.containsKey("security") + env.ARCHITECTURE = bundleManifestObj.getArtifactArchitecture() } } } @@ -74,17 +89,15 @@ pipeline { alwaysPull true } } + when { + environment name: 'HAS_SECURITY', value: 'true' + } steps { script { - def bundleManifestObj = downloadBuildManifest( url: BUNDLE_MANIFEST_URL, path: BUNDLE_MANIFEST ) - String buildId = bundleManifestObj.getArtifactBuildId() - env.BUILD_ID = buildId - env.HAS_SECURITY = bundleManifestObj.components.containsKey("security") - env.ARCHITECTURE = bundleManifestObj.getArtifactArchitecture() echo "BUNDLE_MANIFEST: ${BUNDLE_MANIFEST}" echo "BUILD_ID: ${BUILD_ID}" echo "BUILD_ID: ${BUILD_ID}" @@ -92,17 +105,16 @@ pipeline { echo "Architecture: ${ARCHITECTURE}" lib.jenkins.Messages.new(this).add('perf-test', "Performance tests for #${BUILD_ID}") - if (env.HAS_SECURITY) { - runPerfTestScript(bundleManifest: "${BUNDLE_MANIFEST}", - buildId: "${BUILD_ID}", - architecture: "${ARCHITECTURE}", - insecure: false, - workload: TEST_WORKLOAD, - testIterations: TEST_ITERATIONS, - warmupIterations: WARMUP_ITERATIONS) - lib.jenkins.Messages.new(this).add('perf-test', "Performance tests with security for ${BUILD_ID} completed") - } + runPerfTestScript(bundleManifest: "${BUNDLE_MANIFEST}", + buildId: "${BUILD_ID}", + architecture: "${ARCHITECTURE}", + insecure: false, + workload: TEST_WORKLOAD, + testIterations: TEST_ITERATIONS, + warmupIterations: WARMUP_ITERATIONS) + + lib.jenkins.Messages.new(this).add('perf-test', "Performance tests with security for ${BUILD_ID} completed") } } post { @@ -138,9 +150,7 @@ pipeline { url: BUNDLE_MANIFEST_URL, path: BUNDLE_MANIFEST ) - String buildId = bundleManifestObj.getArtifactBuildId() - env.BUILD_ID = buildId - env.ARCHITECTURE = bundleManifestObj.getArtifactArchitecture() + echo "BUNDLE_MANIFEST: ${BUNDLE_MANIFEST}" echo "BUILD_ID: ${BUILD_ID}" echo "BUILD_ID: ${BUILD_ID}" diff --git a/tests/jenkins/TestRunPerfTestScript.groovy b/tests/jenkins/TestRunPerfTestScript.groovy index 76204281c9..c7204adef7 100644 --- a/tests/jenkins/TestRunPerfTestScript.groovy +++ b/tests/jenkins/TestRunPerfTestScript.groovy @@ -13,9 +13,11 @@ import org.junit.Test class TestRunPerfTestScript extends BuildPipelineTest { + RunPerfTestScriptLibTester perfTestScriptLibTester + @Before void setUp() { - this.registerLibTester(new RunPerfTestScriptLibTester( + perfTestScriptLibTester = new RunPerfTestScriptLibTester( 'tests/jenkins/data/opensearch-1.3.0-bundle.yml', '1236', 'false', @@ -23,12 +25,14 @@ class TestRunPerfTestScript extends BuildPipelineTest { '1', '1' ) - ) + this.registerLibTester(perfTestScriptLibTester) super.setUp() } @Test public void TestRunPerfTestScript() { super.testPipeline("tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile") + perfTestScriptLibTester.setBundleManifest('tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml') + super.testPipeline("tests/jenkins/jobs/RunPerfTestScriptWithoutSecurity_Jenkinsfile") } } diff --git a/tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml b/tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml new file mode 100644 index 0000000000..8a70c63e74 --- /dev/null +++ b/tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml @@ -0,0 +1,16 @@ +--- +schema-version: '1.1' +build: + name: OpenSearch + version: 1.3.0 + platform: linux + architecture: x64 + distribution: tar + location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/1236/linux/x64/dist/opensearch/opensearch-1.3.0-linux-x64.tar.gz + id: '1236' +components: + - name: OpenSearch + repository: https://github.com/opensearch-project/OpenSearch.git + ref: 1.x + commit_id: 22408088f002a4fc8cdd3b2ed7438866c14c5069 + location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/1236/linux/x64/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64.tar.gz diff --git a/tests/jenkins/jobs/RunPerfTestScriptWithoutSecurity_Jenkinsfile b/tests/jenkins/jobs/RunPerfTestScriptWithoutSecurity_Jenkinsfile new file mode 100644 index 0000000000..ff494bc9b5 --- /dev/null +++ b/tests/jenkins/jobs/RunPerfTestScriptWithoutSecurity_Jenkinsfile @@ -0,0 +1,27 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +pipeline { + agent none + stages { + stage('perf-test') { + steps { + script { + runPerfTestScript( + bundleManifest: 'tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml', + buildId: '1236', + insecure: 'false', + workload: 'nyc_taxis', + testIterations: '1', + warmupIterations: '1' + ) + } + } + } + } +} \ No newline at end of file diff --git a/tests/jenkins/jobs/RunPerfTestScriptWithoutSecurity_Jenkinsfile.txt b/tests/jenkins/jobs/RunPerfTestScriptWithoutSecurity_Jenkinsfile.txt new file mode 100644 index 0000000000..a43d86e8f1 --- /dev/null +++ b/tests/jenkins/jobs/RunPerfTestScriptWithoutSecurity_Jenkinsfile.txt @@ -0,0 +1,50 @@ + RunPerfTestScript_Jenkinsfile.run() + RunPerfTestScript_Jenkinsfile.pipeline(groovy.lang.Closure) + RunPerfTestScript_Jenkinsfile.echo(Executing on agent [label:none]) + RunPerfTestScript_Jenkinsfile.stage(perf-test, groovy.lang.Closure) + RunPerfTestScript_Jenkinsfile.script(groovy.lang.Closure) + RunPerfTestScript_Jenkinsfile.runPerfTestScript({bundleManifest=tests/jenkins/data/opensearch-1.3.0-bundle.yml, buildId=1236, insecure=false, workload=nyc_taxis, testIterations=1, warmupIterations=1}) + runPerfTestScript.legacySCM(groovy.lang.Closure) + runPerfTestScript.library({identifier=jenkins@20211123, retriever=null}) + runPerfTestScript.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) + BuildManifest.asBoolean() + runPerfTestScript.sh( + npm install -g fs-extra + npm install -g chalk@4.1.2 + npm install -g @aws-cdk/cloudformation-diff + npm install -g aws-cdk + npm install -g cdk-assume-role-credential-plugin@1.4.0 + ) + runPerfTestScript.sh( + pipenv install "dataclasses_json~=0.5" "aws_requests_auth~=0.4" "json2html~=1.3.0" + pipenv install "aws-cdk.core~=1.143.0" "aws_cdk.aws_ec2~=1.143.0" "aws_cdk.aws_iam~=1.143.0" + pipenv install "boto3~=1.18" "setuptools~=57.4" "retry~=0.9" + ) + runPerfTestScript.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + runPerfTestScript.s3Download({file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true}) + runPerfTestScript.sh(./test.sh perf-test --stack test-single-1236-null --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config config.yml --force-insecure-mode --workload nyc_taxis --test-iters 1 --warmup-iters 1) + RunPerfTestScriptWithoutSecurity_Jenkinsfile.run() + RunPerfTestScriptWithoutSecurity_Jenkinsfile.pipeline(groovy.lang.Closure) + RunPerfTestScriptWithoutSecurity_Jenkinsfile.echo(Executing on agent [label:none]) + RunPerfTestScriptWithoutSecurity_Jenkinsfile.stage(perf-test, groovy.lang.Closure) + RunPerfTestScriptWithoutSecurity_Jenkinsfile.script(groovy.lang.Closure) + RunPerfTestScriptWithoutSecurity_Jenkinsfile.runPerfTestScript({bundleManifest=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml, buildId=1236, insecure=false, workload=nyc_taxis, testIterations=1, warmupIterations=1}) + runPerfTestScript.legacySCM(groovy.lang.Closure) + runPerfTestScript.library({identifier=jenkins@20211123, retriever=null}) + runPerfTestScript.readYaml({file=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) + BuildManifest.asBoolean() + runPerfTestScript.sh( + npm install -g fs-extra + npm install -g chalk@4.1.2 + npm install -g @aws-cdk/cloudformation-diff + npm install -g aws-cdk + npm install -g cdk-assume-role-credential-plugin@1.4.0 + ) + runPerfTestScript.sh( + pipenv install "dataclasses_json~=0.5" "aws_requests_auth~=0.4" "json2html~=1.3.0" + pipenv install "aws-cdk.core~=1.143.0" "aws_cdk.aws_ec2~=1.143.0" "aws_cdk.aws_iam~=1.143.0" + pipenv install "boto3~=1.18" "setuptools~=57.4" "retry~=0.9" + ) + runPerfTestScript.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + runPerfTestScript.s3Download({file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true}) + runPerfTestScript.sh(./test.sh perf-test --stack test-single-1236-null --bundle-manifest tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml --config config.yml --force-insecure-mode --workload nyc_taxis --test-iters 1 --warmup-iters 1) diff --git a/tests/jenkins/lib-testers/RunPerfTestScriptLibTest.groovy b/tests/jenkins/lib-testers/RunPerfTestScriptLibTest.groovy index 912b7e41ca..7b3cb421d4 100644 --- a/tests/jenkins/lib-testers/RunPerfTestScriptLibTest.groovy +++ b/tests/jenkins/lib-testers/RunPerfTestScriptLibTest.groovy @@ -20,6 +20,10 @@ class RunPerfTestScriptLibTester extends LibFunctionTester { this.warmupIterations = warmupIterations } + public void setBundleManifest(bundleManifest) { + this.bundleManifest = bundleManifest + } + void configure(helper, binding) { helper.registerAllowedMethod("s3Download", [Map]) helper.registerAllowedMethod("withAWS", [Map, Closure], { args, closure -> From f9f56742ce7c9984905576759baaae132fd961d5 Mon Sep 17 00:00:00 2001 From: Kunal Kotwani Date: Thu, 24 Mar 2022 22:49:15 -0700 Subject: [PATCH 07/12] Add detailed job tests for performance tests Signed-off-by: Kunal Kotwani --- .../TestRunNonSecurityPerfTestScript.groovy | 117 ++++++++++++ tests/jenkins/TestRunPerfTestScript.groovy | 104 +++++++++-- ...nPerfTestScriptWithoutSecurity_Jenkinsfile | 170 +++++++++++++++++- ...fTestScriptWithoutSecurity_Jenkinsfile.txt | 67 ++++--- .../jobs/RunPerfTestScript_Jenkinsfile | 167 ++++++++++++++++- .../jobs/RunPerfTestScript_Jenkinsfile.txt | 92 +++++++++- .../RunPerfTestScriptLibTest.groovy | 40 +++-- 7 files changed, 683 insertions(+), 74 deletions(-) create mode 100644 tests/jenkins/TestRunNonSecurityPerfTestScript.groovy diff --git a/tests/jenkins/TestRunNonSecurityPerfTestScript.groovy b/tests/jenkins/TestRunNonSecurityPerfTestScript.groovy new file mode 100644 index 0000000000..d153b3c4fc --- /dev/null +++ b/tests/jenkins/TestRunNonSecurityPerfTestScript.groovy @@ -0,0 +1,117 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +import jenkins.tests.BuildPipelineTest +import org.junit.Before +import org.junit.Test + +import static com.lesfurets.jenkins.unit.MethodCall.callArgsToString +import static org.hamcrest.CoreMatchers.equalTo +import static org.hamcrest.CoreMatchers.hasItem +import static org.hamcrest.MatcherAssert.assertThat + +class TestRunSecurityPerfTestScript extends BuildPipelineTest { + + @Before + void setUp() { + this.registerLibTester(new RunPerfTestScriptLibTester( + 'tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml', + '1236', + 'true', + 'nyc_taxis', + '1', + '1', + false + )) + super.setUp() + } + + @Test + public void testRunNonSecurityPerfTestScript_verifyPipeline() { + super.testPipeline("tests/jenkins/jobs/RunPerfTestScriptWithoutSecurity_Jenkinsfile") + } + + @Test + void testRunNonSecurityPerfTestScript_verifyArtifactDownloads() { + runScript('tests/jenkins/jobs/RunPerfTestScriptWithoutSecurity_Jenkinsfile') + + def curlCommands = getCommandExecutions('sh', 'curl').findAll { + shCommand -> shCommand.contains('curl') + } + + assertThat(curlCommands.size(), equalTo(1)) + assertThat(curlCommands, hasItem( + "curl test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml".toString() + )) + + def s3DownloadCommands = getCommandExecutions('s3Download', 'bucket').findAll { + shCommand -> shCommand.contains('bucket') + } + + assertThat(s3DownloadCommands.size(), equalTo(1)) + assertThat(s3DownloadCommands, hasItem( + "{file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true}".toString() + )) + } + + @Test + void testRunNonSecurityPerfTestScript_verifyPackageInstallation() { + runScript('tests/jenkins/jobs/RunPerfTestScriptWithoutSecurity_Jenkinsfile') + + def npmCommands = getCommandExecutions('sh', 'npm').findAll { + shCommand -> shCommand.contains('npm') + } + + assertThat(npmCommands.size(), equalTo(1)) + + def pipenvCommands = getCommandExecutions('sh', 'pipenv').findAll { + shCommand -> shCommand.contains('pipenv') + } + + assertThat(pipenvCommands.size(), equalTo(1)) + + } + + @Test + void testRunNonSecurityPerfTestScript_verifyScriptExecutions() { + runScript('tests/jenkins/jobs/RunPerfTestScriptWithoutSecurity_Jenkinsfile') + + def testScriptCommands = getCommandExecutions('sh', './test.sh').findAll { + shCommand -> shCommand.contains('./test.sh') + } + + assertThat(testScriptCommands.size(), equalTo(1)) + assertThat(testScriptCommands, hasItem( + "./test.sh perf-test --stack test-single-1236-x64 --bundle-manifest tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml --config config.yml --force-insecure-mode --workload nyc_taxis --test-iters 1 --warmup-iters 1".toString() + )) + + def resultUploadScriptCommands = getCommandExecutions('s3Upload', 'test-results').findAll { + shCommand -> shCommand.contains('test-results') + } + assertThat(resultUploadScriptCommands.size(), equalTo(1)) + assertThat(resultUploadScriptCommands, hasItem( + "{file=test-results, bucket=test_bucket, path=perf-test/1.3.0/1236/linux/x64/test-results}".toString() + )) + } + + def getCommandExecutions(methodName, command) { + def shCommands = helper.callStack.findAll { + call -> + call.methodName == methodName + }. + collect { + call -> + callArgsToString(call) + }.findAll { + shCommand -> + shCommand.contains(command) + } + + return shCommands + } +} \ No newline at end of file diff --git a/tests/jenkins/TestRunPerfTestScript.groovy b/tests/jenkins/TestRunPerfTestScript.groovy index c7204adef7..cf8532d134 100644 --- a/tests/jenkins/TestRunPerfTestScript.groovy +++ b/tests/jenkins/TestRunPerfTestScript.groovy @@ -10,29 +10,111 @@ import jenkins.tests.BuildPipelineTest import org.junit.Before import org.junit.Test +import static com.lesfurets.jenkins.unit.MethodCall.callArgsToString +import static org.hamcrest.CoreMatchers.equalTo +import static org.hamcrest.CoreMatchers.hasItem +import static org.hamcrest.MatcherAssert.assertThat class TestRunPerfTestScript extends BuildPipelineTest { - RunPerfTestScriptLibTester perfTestScriptLibTester - @Before void setUp() { - perfTestScriptLibTester = new RunPerfTestScriptLibTester( + this.registerLibTester(new RunPerfTestScriptLibTester( 'tests/jenkins/data/opensearch-1.3.0-bundle.yml', '1236', - 'false', + 'true', 'nyc_taxis', '1', - '1' - ) - this.registerLibTester(perfTestScriptLibTester) + '1', + true + )) super.setUp() } @Test - public void TestRunPerfTestScript() { + public void testRunPerfTestScript_Pipeline() { super.testPipeline("tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile") - perfTestScriptLibTester.setBundleManifest('tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml') - super.testPipeline("tests/jenkins/jobs/RunPerfTestScriptWithoutSecurity_Jenkinsfile") } -} + + @Test + void testRunPerfTestScript_verifyArtifactDownloads() { + runScript('tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile') + + def curlCommands = getCommandExecutions('sh', 'curl').findAll { + shCommand -> shCommand.contains('curl') + } + + assertThat(curlCommands.size(), equalTo(2)) + assertThat(curlCommands, hasItem( + "curl test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-bundle.yml".toString() + )) + + def s3DownloadCommands = getCommandExecutions('s3Download', 'bucket').findAll { + shCommand -> shCommand.contains('bucket') + } + + assertThat(s3DownloadCommands.size(), equalTo(2)) + assertThat(s3DownloadCommands, hasItem( + "{file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true}".toString() + )) + } + + @Test + void testRunPerfTestScript_verifyPackageInstallation() { + runScript('tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile') + + def npmCommands = getCommandExecutions('sh', 'npm').findAll { + shCommand -> shCommand.contains('npm') + } + + assertThat(npmCommands.size(), equalTo(2)) + + def pipenvCommands = getCommandExecutions('sh', 'pipenv').findAll { + shCommand -> shCommand.contains('pipenv') + } + + assertThat(pipenvCommands.size(), equalTo(2)) + + } + + @Test + void testRunPerfTestScript_verifyScriptExecutions() { + runScript('tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile') + + def testScriptCommands = getCommandExecutions('sh', './test.sh').findAll { + shCommand -> shCommand.contains('./test.sh') + } + + assertThat(testScriptCommands.size(), equalTo(2)) + assertThat(testScriptCommands, hasItem( + "./test.sh perf-test --stack test-single-security-1236-x64 --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config config.yml --workload nyc_taxis --test-iters 1 --warmup-iters 1".toString() + )) + assertThat(testScriptCommands, hasItem( + "./test.sh perf-test --stack test-single-1236-x64 --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config config.yml --force-insecure-mode --workload nyc_taxis --test-iters 1 --warmup-iters 1".toString() + )) + + def resultUploadScriptCommands = getCommandExecutions('s3Upload', 'test-results').findAll { + shCommand -> shCommand.contains('test-results') + } + assertThat(resultUploadScriptCommands.size(), equalTo(2)) + assertThat(resultUploadScriptCommands, hasItem( + "{file=test-results, bucket=test_bucket, path=perf-test/1.3.0/1236/linux/x64/test-results}".toString() + )) + } + + def getCommandExecutions(methodName, command) { + def shCommands = helper.callStack.findAll { + call -> + call.methodName == methodName + }. + collect { + call -> + callArgsToString(call) + }.findAll { + shCommand -> + shCommand.contains(command) + } + + return shCommands + } +} \ No newline at end of file diff --git a/tests/jenkins/jobs/RunPerfTestScriptWithoutSecurity_Jenkinsfile b/tests/jenkins/jobs/RunPerfTestScriptWithoutSecurity_Jenkinsfile index ff494bc9b5..25629ee195 100644 --- a/tests/jenkins/jobs/RunPerfTestScriptWithoutSecurity_Jenkinsfile +++ b/tests/jenkins/jobs/RunPerfTestScriptWithoutSecurity_Jenkinsfile @@ -6,20 +6,172 @@ * compatible open source license. */ +lib = library(identifier: "jenkins@20211118", retriever: legacySCM(scm)) + pipeline { agent none + options { + timeout(time: 10, unit: 'HOURS') + } + parameters { + string( + name: 'GITHUB_TOKEN', + description: 'Github token for account access.', + trim: true + ) + string( + name: 'BUNDLE_MANIFEST_URL', + description: 'The bundle manifest URL, e.g. https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.2.2/98/linux/x64/builds/opensearch/manifest.yml.', + trim: true + ) + string( + defaultValue: 'nyc_taxis', + name: 'TEST_WORKLOAD', + description: 'The workload name from OpenSearch Benchmark Workloads for Mensor (internal client).', + trim: true + ) + string( + defaultValue: '1', + name: 'TEST_ITERATIONS', + description: 'Number of times to run a workload for Mensor (internal client).', + trim: true + ) + string( + defaultValue: '0', + name: 'WARMUP_ITERATIONS', + description: 'Number of times to run a workload before collecting data for Mensor (internal client).', + trim: true + ) + } + stages { - stage('perf-test') { + stage('validate-and-set-parameters') { + agent { + docker { + label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' + image 'opensearchstaging/ci-runner:ci-runner-centos7-v1' + alwaysPull true + } + } steps { script { - runPerfTestScript( - bundleManifest: 'tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml', - buildId: '1236', - insecure: 'false', - workload: 'nyc_taxis', - testIterations: '1', - warmupIterations: '1' - ) + if (BUNDLE_MANIFEST_URL == '') { + currentBuild.result = 'ABORTED' + error("Performance Tests failed to start. Missing parameter: BUNDLE_MANIFEST_URL.") + } + if (TEST_ITERATIONS != null && !TEST_ITERATIONS.isInteger()) { + currentBuild.result = 'ABORTED' + error("Performance Tests failed to start. Invalid value for parameter: TEST_ITERATIONS. Value should be an integer.") + } + if (WARMUP_ITERATIONS != null && !WARMUP_ITERATIONS.isInteger()) { + currentBuild.result = 'ABORTED' + error("Performance Tests failed to start. Invalid value for parameter: WARMUP_ITERATIONS. Value should be an integer.") + } + echo "Security: ${HAS_SECURITY}" + } + } + } + stage('perf-test') { + parallel { + stage('test-with-security') { + agent { + docker { + label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' + image 'opensearchstaging/ci-runner:ci-runner-centos7-v1' + alwaysPull true + } + } + when { + environment name: 'HAS_SECURITY', value: 'true' + } + steps { + script { + def bundleManifestObj = downloadBuildManifest( + url: BUNDLE_MANIFEST_URL, + path: BUNDLE_MANIFEST + ) + echo "BUNDLE_MANIFEST: ${BUNDLE_MANIFEST}" + echo "BUILD_ID: ${BUILD_ID}" + echo "BUILD_ID: ${BUILD_ID}" + echo "Security: ${HAS_SECURITY}" + echo "Architecture: ${ARCHITECTURE}" + + runPerfTestScript(bundleManifest: "${BUNDLE_MANIFEST}", + buildId: "${BUILD_ID}", + architecture: "${ARCHITECTURE}", + insecure: false, + workload: TEST_WORKLOAD, + testIterations: TEST_ITERATIONS, + warmupIterations: WARMUP_ITERATIONS) + } + } + post { + success { + script { + uploadTestResults( + buildManifestFileName: "${BUNDLE_MANIFEST}", + jobName: 'perf-test', + buildNumber: "${BUILD_ID}" + ) + } + postCleanup() + } + failure { + postCleanup() + } + aborted { + postCleanup() + } + } + } + stage('test-without-security') { + agent { + docker { + label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' + image 'opensearchstaging/ci-runner:ci-runner-centos7-v1' + alwaysPull true + } + } + steps { + script { + def bundleManifestObj = downloadBuildManifest( + url: BUNDLE_MANIFEST_URL, + path: BUNDLE_MANIFEST + ) + + echo "BUNDLE_MANIFEST: ${BUNDLE_MANIFEST}" + echo "BUILD_ID: ${BUILD_ID}" + echo "BUILD_ID: ${BUILD_ID}" + echo "Architecture: ${ARCHITECTURE}" + + + runPerfTestScript(bundleManifest: "${BUNDLE_MANIFEST}", + buildId: "${BUILD_ID}", + architecture: "${ARCHITECTURE}", + insecure: true, + workload: TEST_WORKLOAD, + testIterations: TEST_ITERATIONS, + warmupIterations: WARMUP_ITERATIONS) + } + } + post { + success { + script { + uploadTestResults( + buildManifestFileName: "${BUNDLE_MANIFEST}", + jobName: 'perf-test', + buildNumber: "${BUILD_ID}" + ) + } + postCleanup() + } + failure { + postCleanup() + } + aborted { + postCleanup() + } + } } } } diff --git a/tests/jenkins/jobs/RunPerfTestScriptWithoutSecurity_Jenkinsfile.txt b/tests/jenkins/jobs/RunPerfTestScriptWithoutSecurity_Jenkinsfile.txt index a43d86e8f1..64b7bdbaa7 100644 --- a/tests/jenkins/jobs/RunPerfTestScriptWithoutSecurity_Jenkinsfile.txt +++ b/tests/jenkins/jobs/RunPerfTestScriptWithoutSecurity_Jenkinsfile.txt @@ -1,34 +1,28 @@ - RunPerfTestScript_Jenkinsfile.run() - RunPerfTestScript_Jenkinsfile.pipeline(groovy.lang.Closure) - RunPerfTestScript_Jenkinsfile.echo(Executing on agent [label:none]) - RunPerfTestScript_Jenkinsfile.stage(perf-test, groovy.lang.Closure) - RunPerfTestScript_Jenkinsfile.script(groovy.lang.Closure) - RunPerfTestScript_Jenkinsfile.runPerfTestScript({bundleManifest=tests/jenkins/data/opensearch-1.3.0-bundle.yml, buildId=1236, insecure=false, workload=nyc_taxis, testIterations=1, warmupIterations=1}) - runPerfTestScript.legacySCM(groovy.lang.Closure) - runPerfTestScript.library({identifier=jenkins@20211123, retriever=null}) - runPerfTestScript.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) - BuildManifest.asBoolean() - runPerfTestScript.sh( - npm install -g fs-extra - npm install -g chalk@4.1.2 - npm install -g @aws-cdk/cloudformation-diff - npm install -g aws-cdk - npm install -g cdk-assume-role-credential-plugin@1.4.0 - ) - runPerfTestScript.sh( - pipenv install "dataclasses_json~=0.5" "aws_requests_auth~=0.4" "json2html~=1.3.0" - pipenv install "aws-cdk.core~=1.143.0" "aws_cdk.aws_ec2~=1.143.0" "aws_cdk.aws_iam~=1.143.0" - pipenv install "boto3~=1.18" "setuptools~=57.4" "retry~=0.9" - ) - runPerfTestScript.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - runPerfTestScript.s3Download({file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true}) - runPerfTestScript.sh(./test.sh perf-test --stack test-single-1236-null --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config config.yml --force-insecure-mode --workload nyc_taxis --test-iters 1 --warmup-iters 1) RunPerfTestScriptWithoutSecurity_Jenkinsfile.run() + RunPerfTestScriptWithoutSecurity_Jenkinsfile.legacySCM(groovy.lang.Closure) + RunPerfTestScriptWithoutSecurity_Jenkinsfile.library({identifier=jenkins@20211118, retriever=null}) RunPerfTestScriptWithoutSecurity_Jenkinsfile.pipeline(groovy.lang.Closure) + RunPerfTestScriptWithoutSecurity_Jenkinsfile.timeout({time=10, unit=HOURS}) RunPerfTestScriptWithoutSecurity_Jenkinsfile.echo(Executing on agent [label:none]) - RunPerfTestScriptWithoutSecurity_Jenkinsfile.stage(perf-test, groovy.lang.Closure) + RunPerfTestScriptWithoutSecurity_Jenkinsfile.stage(validate-and-set-parameters, groovy.lang.Closure) + RunPerfTestScriptWithoutSecurity_Jenkinsfile.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) RunPerfTestScriptWithoutSecurity_Jenkinsfile.script(groovy.lang.Closure) - RunPerfTestScriptWithoutSecurity_Jenkinsfile.runPerfTestScript({bundleManifest=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml, buildId=1236, insecure=false, workload=nyc_taxis, testIterations=1, warmupIterations=1}) + RunPerfTestScriptWithoutSecurity_Jenkinsfile.echo(Security: false) + RunPerfTestScriptWithoutSecurity_Jenkinsfile.echo(Skipping stage test-with-security) + RunPerfTestScriptWithoutSecurity_Jenkinsfile.stage(test-without-security, groovy.lang.Closure) + RunPerfTestScriptWithoutSecurity_Jenkinsfile.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) + RunPerfTestScriptWithoutSecurity_Jenkinsfile.script(groovy.lang.Closure) + RunPerfTestScriptWithoutSecurity_Jenkinsfile.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) + downloadBuildManifest.legacySCM(groovy.lang.Closure) + downloadBuildManifest.library({identifier=jenkins@20211123, retriever=null}) + downloadBuildManifest.sh(curl test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml) + downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) + BuildManifest.asBoolean() + RunPerfTestScriptWithoutSecurity_Jenkinsfile.echo(BUNDLE_MANIFEST: tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml) + RunPerfTestScriptWithoutSecurity_Jenkinsfile.echo(BUILD_ID: 1236) + RunPerfTestScriptWithoutSecurity_Jenkinsfile.echo(BUILD_ID: 1236) + RunPerfTestScriptWithoutSecurity_Jenkinsfile.echo(Architecture: x64) + RunPerfTestScriptWithoutSecurity_Jenkinsfile.runPerfTestScript({bundleManifest=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml, buildId=1236, architecture=x64, insecure=true, workload=nyc_taxis, testIterations=1, warmupIterations=1}) runPerfTestScript.legacySCM(groovy.lang.Closure) runPerfTestScript.library({identifier=jenkins@20211123, retriever=null}) runPerfTestScript.readYaml({file=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) @@ -47,4 +41,21 @@ ) runPerfTestScript.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) runPerfTestScript.s3Download({file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true}) - runPerfTestScript.sh(./test.sh perf-test --stack test-single-1236-null --bundle-manifest tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml --config config.yml --force-insecure-mode --workload nyc_taxis --test-iters 1 --warmup-iters 1) + runPerfTestScript.sh(./test.sh perf-test --stack test-single-1236-x64 --bundle-manifest tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml --config config.yml --force-insecure-mode --workload nyc_taxis --test-iters 1 --warmup-iters 1) + RunPerfTestScriptWithoutSecurity_Jenkinsfile.script(groovy.lang.Closure) + RunPerfTestScriptWithoutSecurity_Jenkinsfile.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml, jobName=perf-test, buildNumber=1236}) + uploadTestResults.legacySCM(groovy.lang.Closure) + uploadTestResults.library({identifier=jenkins@20211123, retriever=null}) + uploadTestResults.readYaml({file=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) + BuildManifest.asBoolean() + BuildManifest.getArtifactRoot(perf-test, 1236) + uploadTestResults.echo(Uploading to s3://test_bucket/perf-test/1.3.0/1236/linux/x64) + uploadTestResults.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + uploadTestResults.s3Upload({file=test-results, bucket=test_bucket, path=perf-test/1.3.0/1236/linux/x64/test-results}) + BuildManifest.getArtifactRootUrl(test://artifact.url, perf-test, 1236) + Messages.asBoolean() + Messages.add(test_stage, test://artifact.url/perf-test/1.3.0/1236/linux/x64/test-results/) + uploadTestResults.writeFile({file=messages/test_stage.msg, text=test://artifact.url/perf-test/1.3.0/1236/linux/x64/test-results/}) + uploadTestResults.stash({includes=messages/*, name=messages-test_stage}) + RunPerfTestScriptWithoutSecurity_Jenkinsfile.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) diff --git a/tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile b/tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile index 65f45b6b55..d44b8a37ac 100644 --- a/tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile +++ b/tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile @@ -6,20 +6,169 @@ * compatible open source license. */ +lib = library(identifier: "jenkins@20211118", retriever: legacySCM(scm)) + pipeline { agent none + options { + timeout(time: 10, unit: 'HOURS') + } + parameters { + string( + name: 'GITHUB_TOKEN', + description: 'Github token for account access.', + trim: true + ) + string( + name: 'BUNDLE_MANIFEST_URL', + description: 'The bundle manifest URL, e.g. https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.2.2/98/linux/x64/builds/opensearch/manifest.yml.', + trim: true + ) + string( + defaultValue: 'nyc_taxis', + name: 'TEST_WORKLOAD', + description: 'The workload name from OpenSearch Benchmark Workloads for Mensor (internal client).', + trim: true + ) + string( + defaultValue: '1', + name: 'TEST_ITERATIONS', + description: 'Number of times to run a workload for Mensor (internal client).', + trim: true + ) + string( + defaultValue: '0', + name: 'WARMUP_ITERATIONS', + description: 'Number of times to run a workload before collecting data for Mensor (internal client).', + trim: true + ) + } + stages { - stage('perf-test') { + stage('validate-and-set-parameters') { + agent { + docker { + label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' + image 'opensearchstaging/ci-runner:ci-runner-centos7-v1' + alwaysPull true + } + } steps { script { - runPerfTestScript( - bundleManifest: 'tests/jenkins/data/opensearch-1.3.0-bundle.yml', - buildId: '1236', - insecure: 'false', - workload: 'nyc_taxis', - testIterations: '1', - warmupIterations: '1' - ) + if (BUNDLE_MANIFEST_URL == '') { + currentBuild.result = 'ABORTED' + error("Performance Tests failed to start. Missing parameter: BUNDLE_MANIFEST_URL.") + } + if (TEST_ITERATIONS != null && !TEST_ITERATIONS.isInteger()) { + currentBuild.result = 'ABORTED' + error("Performance Tests failed to start. Invalid value for parameter: TEST_ITERATIONS. Value should be an integer.") + } + if (WARMUP_ITERATIONS != null && !WARMUP_ITERATIONS.isInteger()) { + currentBuild.result = 'ABORTED' + error("Performance Tests failed to start. Invalid value for parameter: WARMUP_ITERATIONS. Value should be an integer.") + } + echo "Security: ${HAS_SECURITY}" + } + } + } + stage('perf-test') { + parallel { + stage('test-with-security') { + agent { + docker { + label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' + image 'opensearchstaging/ci-runner:ci-runner-centos7-v1' + alwaysPull true + } + } + steps { + script { + def bundleManifestObj = downloadBuildManifest( + url: BUNDLE_MANIFEST_URL, + path: BUNDLE_MANIFEST + ) + echo "BUNDLE_MANIFEST: ${BUNDLE_MANIFEST}" + echo "BUILD_ID: ${BUILD_ID}" + echo "BUILD_ID: ${BUILD_ID}" + echo "Security: ${HAS_SECURITY}" + echo "Architecture: ${ARCHITECTURE}" + + runPerfTestScript(bundleManifest: "${BUNDLE_MANIFEST}", + buildId: "${BUILD_ID}", + architecture: "${ARCHITECTURE}", + insecure: false, + workload: TEST_WORKLOAD, + testIterations: TEST_ITERATIONS, + warmupIterations: WARMUP_ITERATIONS) + } + } + post { + success { + script { + uploadTestResults( + buildManifestFileName: "${BUNDLE_MANIFEST}", + jobName: 'perf-test', + buildNumber: "${BUILD_ID}" + ) + } + postCleanup() + } + failure { + postCleanup() + } + aborted { + postCleanup() + } + } + } + stage('test-without-security') { + agent { + docker { + label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' + image 'opensearchstaging/ci-runner:ci-runner-centos7-v1' + alwaysPull true + } + } + steps { + script { + def bundleManifestObj = downloadBuildManifest( + url: BUNDLE_MANIFEST_URL, + path: BUNDLE_MANIFEST + ) + + echo "BUNDLE_MANIFEST: ${BUNDLE_MANIFEST}" + echo "BUILD_ID: ${BUILD_ID}" + echo "BUILD_ID: ${BUILD_ID}" + echo "Architecture: ${ARCHITECTURE}" + + + runPerfTestScript(bundleManifest: "${BUNDLE_MANIFEST}", + buildId: "${BUILD_ID}", + architecture: "${ARCHITECTURE}", + insecure: true, + workload: TEST_WORKLOAD, + testIterations: TEST_ITERATIONS, + warmupIterations: WARMUP_ITERATIONS) + } + } + post { + success { + script { + uploadTestResults( + buildManifestFileName: "${BUNDLE_MANIFEST}", + jobName: 'perf-test', + buildNumber: "${BUILD_ID}" + ) + } + postCleanup() + } + failure { + postCleanup() + } + aborted { + postCleanup() + } + } } } } diff --git a/tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile.txt b/tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile.txt index 12762e3bf1..769c03e0ab 100644 --- a/tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile.txt +++ b/tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile.txt @@ -1,9 +1,78 @@ RunPerfTestScript_Jenkinsfile.run() + RunPerfTestScript_Jenkinsfile.legacySCM(groovy.lang.Closure) + RunPerfTestScript_Jenkinsfile.library({identifier=jenkins@20211118, retriever=null}) RunPerfTestScript_Jenkinsfile.pipeline(groovy.lang.Closure) + RunPerfTestScript_Jenkinsfile.timeout({time=10, unit=HOURS}) RunPerfTestScript_Jenkinsfile.echo(Executing on agent [label:none]) - RunPerfTestScript_Jenkinsfile.stage(perf-test, groovy.lang.Closure) + RunPerfTestScript_Jenkinsfile.stage(validate-and-set-parameters, groovy.lang.Closure) + RunPerfTestScript_Jenkinsfile.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) RunPerfTestScript_Jenkinsfile.script(groovy.lang.Closure) - RunPerfTestScript_Jenkinsfile.runPerfTestScript({bundleManifest=tests/jenkins/data/opensearch-1.3.0-bundle.yml, buildId=1236, insecure=false, workload=nyc_taxis, testIterations=1, warmupIterations=1}) + RunPerfTestScript_Jenkinsfile.echo(Security: true) + RunPerfTestScript_Jenkinsfile.stage(test-with-security, groovy.lang.Closure) + RunPerfTestScript_Jenkinsfile.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) + RunPerfTestScript_Jenkinsfile.script(groovy.lang.Closure) + RunPerfTestScript_Jenkinsfile.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) + downloadBuildManifest.legacySCM(groovy.lang.Closure) + downloadBuildManifest.library({identifier=jenkins@20211123, retriever=null}) + downloadBuildManifest.sh(curl test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-bundle.yml) + downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) + BuildManifest.asBoolean() + RunPerfTestScript_Jenkinsfile.echo(BUNDLE_MANIFEST: tests/jenkins/data/opensearch-1.3.0-bundle.yml) + RunPerfTestScript_Jenkinsfile.echo(BUILD_ID: 1236) + RunPerfTestScript_Jenkinsfile.echo(BUILD_ID: 1236) + RunPerfTestScript_Jenkinsfile.echo(Security: true) + RunPerfTestScript_Jenkinsfile.echo(Architecture: x64) + RunPerfTestScript_Jenkinsfile.runPerfTestScript({bundleManifest=tests/jenkins/data/opensearch-1.3.0-bundle.yml, buildId=1236, architecture=x64, insecure=false, workload=nyc_taxis, testIterations=1, warmupIterations=1}) + runPerfTestScript.legacySCM(groovy.lang.Closure) + runPerfTestScript.library({identifier=jenkins@20211123, retriever=null}) + runPerfTestScript.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) + BuildManifest.asBoolean() + runPerfTestScript.sh( + npm install -g fs-extra + npm install -g chalk@4.1.2 + npm install -g @aws-cdk/cloudformation-diff + npm install -g aws-cdk + npm install -g cdk-assume-role-credential-plugin@1.4.0 + ) + runPerfTestScript.sh( + pipenv install "dataclasses_json~=0.5" "aws_requests_auth~=0.4" "json2html~=1.3.0" + pipenv install "aws-cdk.core~=1.143.0" "aws_cdk.aws_ec2~=1.143.0" "aws_cdk.aws_iam~=1.143.0" + pipenv install "boto3~=1.18" "setuptools~=57.4" "retry~=0.9" + ) + runPerfTestScript.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + runPerfTestScript.s3Download({file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true}) + runPerfTestScript.sh(./test.sh perf-test --stack test-single-security-1236-x64 --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config config.yml --workload nyc_taxis --test-iters 1 --warmup-iters 1) + RunPerfTestScript_Jenkinsfile.script(groovy.lang.Closure) + RunPerfTestScript_Jenkinsfile.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-1.3.0-bundle.yml, jobName=perf-test, buildNumber=1236}) + uploadTestResults.legacySCM(groovy.lang.Closure) + uploadTestResults.library({identifier=jenkins@20211123, retriever=null}) + uploadTestResults.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) + BuildManifest.asBoolean() + BuildManifest.getArtifactRoot(perf-test, 1236) + uploadTestResults.echo(Uploading to s3://test_bucket/perf-test/1.3.0/1236/linux/x64) + uploadTestResults.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + uploadTestResults.s3Upload({file=test-results, bucket=test_bucket, path=perf-test/1.3.0/1236/linux/x64/test-results}) + BuildManifest.getArtifactRootUrl(test://artifact.url, perf-test, 1236) + Messages.asBoolean() + Messages.add(test_stage, test://artifact.url/perf-test/1.3.0/1236/linux/x64/test-results/) + uploadTestResults.writeFile({file=messages/test_stage.msg, text=test://artifact.url/perf-test/1.3.0/1236/linux/x64/test-results/}) + uploadTestResults.stash({includes=messages/*, name=messages-test_stage}) + RunPerfTestScript_Jenkinsfile.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) + RunPerfTestScript_Jenkinsfile.stage(test-without-security, groovy.lang.Closure) + RunPerfTestScript_Jenkinsfile.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) + RunPerfTestScript_Jenkinsfile.script(groovy.lang.Closure) + RunPerfTestScript_Jenkinsfile.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) + downloadBuildManifest.legacySCM(groovy.lang.Closure) + downloadBuildManifest.library({identifier=jenkins@20211123, retriever=null}) + downloadBuildManifest.sh(curl test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-bundle.yml) + downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) + BuildManifest.asBoolean() + RunPerfTestScript_Jenkinsfile.echo(BUNDLE_MANIFEST: tests/jenkins/data/opensearch-1.3.0-bundle.yml) + RunPerfTestScript_Jenkinsfile.echo(BUILD_ID: 1236) + RunPerfTestScript_Jenkinsfile.echo(BUILD_ID: 1236) + RunPerfTestScript_Jenkinsfile.echo(Architecture: x64) + RunPerfTestScript_Jenkinsfile.runPerfTestScript({bundleManifest=tests/jenkins/data/opensearch-1.3.0-bundle.yml, buildId=1236, architecture=x64, insecure=true, workload=nyc_taxis, testIterations=1, warmupIterations=1}) runPerfTestScript.legacySCM(groovy.lang.Closure) runPerfTestScript.library({identifier=jenkins@20211123, retriever=null}) runPerfTestScript.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) @@ -22,4 +91,21 @@ ) runPerfTestScript.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) runPerfTestScript.s3Download({file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true}) - runPerfTestScript.sh(./test.sh perf-test --stack test-single-1236-null --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config config.yml --force-insecure-mode --workload nyc_taxis --test-iters 1 --warmup-iters 1) + runPerfTestScript.sh(./test.sh perf-test --stack test-single-1236-x64 --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config config.yml --force-insecure-mode --workload nyc_taxis --test-iters 1 --warmup-iters 1) + RunPerfTestScript_Jenkinsfile.script(groovy.lang.Closure) + RunPerfTestScript_Jenkinsfile.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-1.3.0-bundle.yml, jobName=perf-test, buildNumber=1236}) + uploadTestResults.legacySCM(groovy.lang.Closure) + uploadTestResults.library({identifier=jenkins@20211123, retriever=null}) + uploadTestResults.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) + BuildManifest.asBoolean() + BuildManifest.getArtifactRoot(perf-test, 1236) + uploadTestResults.echo(Uploading to s3://test_bucket/perf-test/1.3.0/1236/linux/x64) + uploadTestResults.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + uploadTestResults.s3Upload({file=test-results, bucket=test_bucket, path=perf-test/1.3.0/1236/linux/x64/test-results}) + BuildManifest.getArtifactRootUrl(test://artifact.url, perf-test, 1236) + Messages.asBoolean() + Messages.add(test_stage, test://artifact.url/perf-test/1.3.0/1236/linux/x64/test-results/) + uploadTestResults.writeFile({file=messages/test_stage.msg, text=test://artifact.url/perf-test/1.3.0/1236/linux/x64/test-results/}) + uploadTestResults.stash({includes=messages/*, name=messages-test_stage}) + RunPerfTestScript_Jenkinsfile.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) diff --git a/tests/jenkins/lib-testers/RunPerfTestScriptLibTest.groovy b/tests/jenkins/lib-testers/RunPerfTestScriptLibTest.groovy index 7b3cb421d4..498615c487 100644 --- a/tests/jenkins/lib-testers/RunPerfTestScriptLibTest.groovy +++ b/tests/jenkins/lib-testers/RunPerfTestScriptLibTest.groovy @@ -1,7 +1,6 @@ import static org.hamcrest.CoreMatchers.notNullValue import static org.hamcrest.MatcherAssert.assertThat - class RunPerfTestScriptLibTester extends LibFunctionTester { private String bundleManifest @@ -10,33 +9,47 @@ class RunPerfTestScriptLibTester extends LibFunctionTester { private String workload private String testIterations private String warmupIterations + private boolean security - public RunPerfTestScriptLibTester(bundleManifest, buildId, insecure, workload, testIterations, warmupIterations){ + public RunPerfTestScriptLibTester(bundleManifest, buildId, insecure, workload, + testIterations, warmupIterations, security) { this.bundleManifest = bundleManifest this.buildId = buildId this.insecure = insecure this.workload = workload this.testIterations = testIterations this.warmupIterations = warmupIterations - } - - public void setBundleManifest(bundleManifest) { - this.bundleManifest = bundleManifest + this.security = security } void configure(helper, binding) { helper.registerAllowedMethod("s3Download", [Map]) - helper.registerAllowedMethod("withAWS", [Map, Closure], { args, closure -> + helper.registerAllowedMethod("uploadTestResults", [Map]) + helper.registerAllowedMethod("s3Upload", [Map]) + helper.registerAllowedMethod("withAWS", [Map, Closure], { + args, + closure -> closure.delegate = delegate return helper.callClosure(closure) }) binding.setVariable('GITHUB_TOKEN', 'test_token') binding.setVariable('ARTIFACT_BUCKET_NAME', 'test_bucket') + binding.setVariable('PUBLIC_ARTIFACT_URL', 'test://artifact.url') + binding.setVariable('STAGE_NAME', 'test_stage') binding.setVariable('PERF_TEST_CONFIG_LOCATION', 'test_config') binding.setVariable('ARTIFACT_DOWNLOAD_ROLE_NAME', 'Dummy_Download_Role') binding.setVariable('AWS_ACCOUNT_PUBLIC', 'dummy_account') + binding.setVariable('BUNDLE_MANIFEST_URL', 'test://artifact.url') + binding.setVariable('BUNDLE_MANIFEST', bundleManifest) + binding.setVariable('BUILD_ID', buildId) + binding.setVariable('HAS_SECURITY', security) + binding.setVariable('TEST_ITERATIONS', testIterations) + binding.setVariable('WARMUP_ITERATIONS', warmupIterations) + binding.setVariable('TEST_WORKLOAD', workload) + binding.setVariable('ARCHITECTURE', 'x64') + } void parameterInvariantsAssertions(call) { @@ -49,15 +62,14 @@ class RunPerfTestScriptLibTester extends LibFunctionTester { } boolean expectedParametersMatcher(call) { - return call.args.bundleManifest.first().toString().equals(this.bundleManifest) - && call.args.buildId.first().toString().equals(this.buildId) - && call.args.insecure.first().toString().equals(this.insecure) - && call.args.workload.first().toString().equals(this.workload) - && call.args.testIterations.first().toInteger().equals(this.testIterations.toInteger()) - && call.args.warmupIterations.first().toInteger().equals(this.warmupIterations.toInteger()) + return call.args.bundleManifest.first().toString().equals(this.bundleManifest) && + call.args.buildId.first().toString().equals(this.buildId) && + call.args.workload.first().toString().equals(this.workload) && + call.args.testIterations.first().toInteger().equals(this.testIterations.toInteger()) && + call.args.warmupIterations.first().toInteger().equals(this.warmupIterations.toInteger()) } String libFunctionName() { return 'runPerfTestScript' } -} +} \ No newline at end of file From c01ea403009a900cf2854988aae610ee7879946a Mon Sep 17 00:00:00 2001 From: Kunal Kotwani Date: Thu, 24 Mar 2022 22:59:42 -0700 Subject: [PATCH 08/12] Fix agent label for notifications Signed-off-by: Kunal Kotwani --- jenkins/opensearch/perf-test.jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jenkins/opensearch/perf-test.jenkinsfile b/jenkins/opensearch/perf-test.jenkinsfile index a3e2a95663..748a8774dc 100644 --- a/jenkins/opensearch/perf-test.jenkinsfile +++ b/jenkins/opensearch/perf-test.jenkinsfile @@ -8,7 +8,7 @@ pipeline { environment { AGENT_LABEL = 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' AGENT_IMAGE = 'opensearchstaging/ci-runner:ci-runner-centos7-v1' - BUNDLE_MANIFEST = "bundle-manifest.yml" + BUNDLE_MANIFEST = 'bundle-manifest.yml' } parameters { string( @@ -194,7 +194,7 @@ pipeline { post { success { - node(AGENT_X64) { + node(AGENT_LABEL) { script { def stashed = lib.jenkins.Messages.new(this).get(['perf-test']) publishNotification( @@ -209,7 +209,7 @@ pipeline { } } failure { - node(AGENT_X64) { + node(AGENT_LABEL) { script { def stashed = lib.jenkins.Messages.new(this).get(['perf-test']) publishNotification( From 74bf99ad2658559ae56d6c644ed1a868cc691056 Mon Sep 17 00:00:00 2001 From: Kunal Kotwani Date: Fri, 25 Mar 2022 15:06:14 -0700 Subject: [PATCH 09/12] Run tests against actual Jenkins perf-test job Signed-off-by: Kunal Kotwani --- jenkins/opensearch/perf-test.jenkinsfile | 5 +- .../TestRunNonSecurityPerfTestScript.groovy | 11 +- tests/jenkins/TestRunPerfTestScript.groovy | 11 +- .../perf-test-with-security.jenkinsfile.txt} | 106 +++++++---- .../opensearch/perf-test.jenkinsfile.txt | 91 +++++++++ ...nPerfTestScriptWithoutSecurity_Jenkinsfile | 179 ------------------ ...fTestScriptWithoutSecurity_Jenkinsfile.txt | 61 ------ .../jobs/RunPerfTestScript_Jenkinsfile | 176 ----------------- .../RunPerfTestScriptLibTest.groovy | 28 ++- 9 files changed, 194 insertions(+), 474 deletions(-) rename tests/jenkins/{jobs/RunPerfTestScript_Jenkinsfile.txt => jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile.txt} (51%) create mode 100644 tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt delete mode 100644 tests/jenkins/jobs/RunPerfTestScriptWithoutSecurity_Jenkinsfile delete mode 100644 tests/jenkins/jobs/RunPerfTestScriptWithoutSecurity_Jenkinsfile.txt delete mode 100644 tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile diff --git a/jenkins/opensearch/perf-test.jenkinsfile b/jenkins/opensearch/perf-test.jenkinsfile index 748a8774dc..fa96a95208 100644 --- a/jenkins/opensearch/perf-test.jenkinsfile +++ b/jenkins/opensearch/perf-test.jenkinsfile @@ -90,7 +90,7 @@ pipeline { } } when { - environment name: 'HAS_SECURITY', value: 'true' + expression { HAS_SECURITY == true } } steps { script { @@ -100,8 +100,6 @@ pipeline { ) echo "BUNDLE_MANIFEST: ${BUNDLE_MANIFEST}" echo "BUILD_ID: ${BUILD_ID}" - echo "BUILD_ID: ${BUILD_ID}" - echo "Security: ${HAS_SECURITY}" echo "Architecture: ${ARCHITECTURE}" lib.jenkins.Messages.new(this).add('perf-test', "Performance tests for #${BUILD_ID}") @@ -153,7 +151,6 @@ pipeline { echo "BUNDLE_MANIFEST: ${BUNDLE_MANIFEST}" echo "BUILD_ID: ${BUILD_ID}" - echo "BUILD_ID: ${BUILD_ID}" echo "Architecture: ${ARCHITECTURE}" lib.jenkins.Messages.new(this).add('perf-test', "Performance tests for #${BUILD_ID}") diff --git a/tests/jenkins/TestRunNonSecurityPerfTestScript.groovy b/tests/jenkins/TestRunNonSecurityPerfTestScript.groovy index d153b3c4fc..387ac12076 100644 --- a/tests/jenkins/TestRunNonSecurityPerfTestScript.groovy +++ b/tests/jenkins/TestRunNonSecurityPerfTestScript.groovy @@ -33,18 +33,19 @@ class TestRunSecurityPerfTestScript extends BuildPipelineTest { @Test public void testRunNonSecurityPerfTestScript_verifyPipeline() { - super.testPipeline("tests/jenkins/jobs/RunPerfTestScriptWithoutSecurity_Jenkinsfile") + super.testPipeline("jenkins/opensearch/perf-test.jenkinsfile", + "tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile") } @Test void testRunNonSecurityPerfTestScript_verifyArtifactDownloads() { - runScript('tests/jenkins/jobs/RunPerfTestScriptWithoutSecurity_Jenkinsfile') + runScript("jenkins/opensearch/perf-test.jenkinsfile") def curlCommands = getCommandExecutions('sh', 'curl').findAll { shCommand -> shCommand.contains('curl') } - assertThat(curlCommands.size(), equalTo(1)) + assertThat(curlCommands.size(), equalTo(3)) assertThat(curlCommands, hasItem( "curl test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml".toString() )) @@ -61,7 +62,7 @@ class TestRunSecurityPerfTestScript extends BuildPipelineTest { @Test void testRunNonSecurityPerfTestScript_verifyPackageInstallation() { - runScript('tests/jenkins/jobs/RunPerfTestScriptWithoutSecurity_Jenkinsfile') + runScript("jenkins/opensearch/perf-test.jenkinsfile") def npmCommands = getCommandExecutions('sh', 'npm').findAll { shCommand -> shCommand.contains('npm') @@ -79,7 +80,7 @@ class TestRunSecurityPerfTestScript extends BuildPipelineTest { @Test void testRunNonSecurityPerfTestScript_verifyScriptExecutions() { - runScript('tests/jenkins/jobs/RunPerfTestScriptWithoutSecurity_Jenkinsfile') + runScript("jenkins/opensearch/perf-test.jenkinsfile") def testScriptCommands = getCommandExecutions('sh', './test.sh').findAll { shCommand -> shCommand.contains('./test.sh') diff --git a/tests/jenkins/TestRunPerfTestScript.groovy b/tests/jenkins/TestRunPerfTestScript.groovy index cf8532d134..ab86b43630 100644 --- a/tests/jenkins/TestRunPerfTestScript.groovy +++ b/tests/jenkins/TestRunPerfTestScript.groovy @@ -33,18 +33,19 @@ class TestRunPerfTestScript extends BuildPipelineTest { @Test public void testRunPerfTestScript_Pipeline() { - super.testPipeline("tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile") + super.testPipeline("jenkins/opensearch/perf-test.jenkinsfile", + "tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile") } @Test void testRunPerfTestScript_verifyArtifactDownloads() { - runScript('tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile') + runScript("jenkins/opensearch/perf-test.jenkinsfile") def curlCommands = getCommandExecutions('sh', 'curl').findAll { shCommand -> shCommand.contains('curl') } - assertThat(curlCommands.size(), equalTo(2)) + assertThat(curlCommands.size(), equalTo(4)) assertThat(curlCommands, hasItem( "curl test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-bundle.yml".toString() )) @@ -61,7 +62,7 @@ class TestRunPerfTestScript extends BuildPipelineTest { @Test void testRunPerfTestScript_verifyPackageInstallation() { - runScript('tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile') + runScript("jenkins/opensearch/perf-test.jenkinsfile") def npmCommands = getCommandExecutions('sh', 'npm').findAll { shCommand -> shCommand.contains('npm') @@ -79,7 +80,7 @@ class TestRunPerfTestScript extends BuildPipelineTest { @Test void testRunPerfTestScript_verifyScriptExecutions() { - runScript('tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile') + runScript("jenkins/opensearch/perf-test.jenkinsfile") def testScriptCommands = getCommandExecutions('sh', './test.sh').findAll { shCommand -> shCommand.contains('./test.sh') diff --git a/tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile.txt similarity index 51% rename from tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile.txt rename to tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile.txt index 769c03e0ab..9aa9d84d95 100644 --- a/tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile.txt @@ -1,28 +1,37 @@ - RunPerfTestScript_Jenkinsfile.run() - RunPerfTestScript_Jenkinsfile.legacySCM(groovy.lang.Closure) - RunPerfTestScript_Jenkinsfile.library({identifier=jenkins@20211118, retriever=null}) - RunPerfTestScript_Jenkinsfile.pipeline(groovy.lang.Closure) - RunPerfTestScript_Jenkinsfile.timeout({time=10, unit=HOURS}) - RunPerfTestScript_Jenkinsfile.echo(Executing on agent [label:none]) - RunPerfTestScript_Jenkinsfile.stage(validate-and-set-parameters, groovy.lang.Closure) - RunPerfTestScript_Jenkinsfile.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) - RunPerfTestScript_Jenkinsfile.script(groovy.lang.Closure) - RunPerfTestScript_Jenkinsfile.echo(Security: true) - RunPerfTestScript_Jenkinsfile.stage(test-with-security, groovy.lang.Closure) - RunPerfTestScript_Jenkinsfile.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) - RunPerfTestScript_Jenkinsfile.script(groovy.lang.Closure) - RunPerfTestScript_Jenkinsfile.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) + perf-test.run() + perf-test.legacySCM(groovy.lang.Closure) + perf-test.library({identifier=jenkins@20211118, retriever=null}) + perf-test.pipeline(groovy.lang.Closure) + perf-test.timeout({time=10, unit=HOURS}) + perf-test.echo(Executing on agent [label:none]) + perf-test.stage(validate-and-set-parameters, groovy.lang.Closure) + perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) + perf-test.script(groovy.lang.Closure) + perf-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) downloadBuildManifest.library({identifier=jenkins@20211123, retriever=null}) downloadBuildManifest.sh(curl test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-bundle.yml) downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) BuildManifest.asBoolean() - RunPerfTestScript_Jenkinsfile.echo(BUNDLE_MANIFEST: tests/jenkins/data/opensearch-1.3.0-bundle.yml) - RunPerfTestScript_Jenkinsfile.echo(BUILD_ID: 1236) - RunPerfTestScript_Jenkinsfile.echo(BUILD_ID: 1236) - RunPerfTestScript_Jenkinsfile.echo(Security: true) - RunPerfTestScript_Jenkinsfile.echo(Architecture: x64) - RunPerfTestScript_Jenkinsfile.runPerfTestScript({bundleManifest=tests/jenkins/data/opensearch-1.3.0-bundle.yml, buildId=1236, architecture=x64, insecure=false, workload=nyc_taxis, testIterations=1, warmupIterations=1}) + BuildManifest.getArtifactBuildId() + BuildManifest.getArtifactArchitecture() + perf-test.stage(test-with-security, groovy.lang.Closure) + perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) + perf-test.script(groovy.lang.Closure) + perf-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) + downloadBuildManifest.legacySCM(groovy.lang.Closure) + downloadBuildManifest.library({identifier=jenkins@20211123, retriever=null}) + downloadBuildManifest.sh(curl test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-bundle.yml) + downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) + BuildManifest.asBoolean() + perf-test.echo(BUNDLE_MANIFEST: tests/jenkins/data/opensearch-1.3.0-bundle.yml) + perf-test.echo(BUILD_ID: 1236) + perf-test.echo(Architecture: x64) + Messages.asBoolean() + Messages.add(perf-test, Performance tests for #1236) + perf-test.writeFile({file=messages/perf-test.msg, text=Performance tests for #1236}) + perf-test.stash({includes=messages/*, name=messages-perf-test}) + perf-test.runPerfTestScript({bundleManifest=tests/jenkins/data/opensearch-1.3.0-bundle.yml, buildId=1236, architecture=x64, insecure=false, workload=nyc_taxis, testIterations=1, warmupIterations=1}) runPerfTestScript.legacySCM(groovy.lang.Closure) runPerfTestScript.library({identifier=jenkins@20211123, retriever=null}) runPerfTestScript.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) @@ -42,8 +51,12 @@ runPerfTestScript.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) runPerfTestScript.s3Download({file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true}) runPerfTestScript.sh(./test.sh perf-test --stack test-single-security-1236-x64 --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config config.yml --workload nyc_taxis --test-iters 1 --warmup-iters 1) - RunPerfTestScript_Jenkinsfile.script(groovy.lang.Closure) - RunPerfTestScript_Jenkinsfile.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-1.3.0-bundle.yml, jobName=perf-test, buildNumber=1236}) + Messages.asBoolean() + Messages.add(perf-test, Performance tests with security for 1236 completed) + perf-test.writeFile({file=messages/perf-test.msg, text=Performance tests with security for 1236 completed}) + perf-test.stash({includes=messages/*, name=messages-perf-test}) + perf-test.script(groovy.lang.Closure) + perf-test.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-1.3.0-bundle.yml, jobName=perf-test, buildNumber=1236}) uploadTestResults.legacySCM(groovy.lang.Closure) uploadTestResults.library({identifier=jenkins@20211123, retriever=null}) uploadTestResults.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) @@ -57,22 +70,25 @@ Messages.add(test_stage, test://artifact.url/perf-test/1.3.0/1236/linux/x64/test-results/) uploadTestResults.writeFile({file=messages/test_stage.msg, text=test://artifact.url/perf-test/1.3.0/1236/linux/x64/test-results/}) uploadTestResults.stash({includes=messages/*, name=messages-test_stage}) - RunPerfTestScript_Jenkinsfile.postCleanup() + perf-test.postCleanup() postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) - RunPerfTestScript_Jenkinsfile.stage(test-without-security, groovy.lang.Closure) - RunPerfTestScript_Jenkinsfile.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) - RunPerfTestScript_Jenkinsfile.script(groovy.lang.Closure) - RunPerfTestScript_Jenkinsfile.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) + perf-test.stage(test-without-security, groovy.lang.Closure) + perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) + perf-test.script(groovy.lang.Closure) + perf-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) downloadBuildManifest.library({identifier=jenkins@20211123, retriever=null}) downloadBuildManifest.sh(curl test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-bundle.yml) downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) BuildManifest.asBoolean() - RunPerfTestScript_Jenkinsfile.echo(BUNDLE_MANIFEST: tests/jenkins/data/opensearch-1.3.0-bundle.yml) - RunPerfTestScript_Jenkinsfile.echo(BUILD_ID: 1236) - RunPerfTestScript_Jenkinsfile.echo(BUILD_ID: 1236) - RunPerfTestScript_Jenkinsfile.echo(Architecture: x64) - RunPerfTestScript_Jenkinsfile.runPerfTestScript({bundleManifest=tests/jenkins/data/opensearch-1.3.0-bundle.yml, buildId=1236, architecture=x64, insecure=true, workload=nyc_taxis, testIterations=1, warmupIterations=1}) + perf-test.echo(BUNDLE_MANIFEST: tests/jenkins/data/opensearch-1.3.0-bundle.yml) + perf-test.echo(BUILD_ID: 1236) + perf-test.echo(Architecture: x64) + Messages.asBoolean() + Messages.add(perf-test, Performance tests for #1236) + perf-test.writeFile({file=messages/perf-test.msg, text=Performance tests for #1236}) + perf-test.stash({includes=messages/*, name=messages-perf-test}) + perf-test.runPerfTestScript({bundleManifest=tests/jenkins/data/opensearch-1.3.0-bundle.yml, buildId=1236, architecture=x64, insecure=true, workload=nyc_taxis, testIterations=1, warmupIterations=1}) runPerfTestScript.legacySCM(groovy.lang.Closure) runPerfTestScript.library({identifier=jenkins@20211123, retriever=null}) runPerfTestScript.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) @@ -92,8 +108,12 @@ runPerfTestScript.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) runPerfTestScript.s3Download({file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true}) runPerfTestScript.sh(./test.sh perf-test --stack test-single-1236-x64 --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config config.yml --force-insecure-mode --workload nyc_taxis --test-iters 1 --warmup-iters 1) - RunPerfTestScript_Jenkinsfile.script(groovy.lang.Closure) - RunPerfTestScript_Jenkinsfile.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-1.3.0-bundle.yml, jobName=perf-test, buildNumber=1236}) + Messages.asBoolean() + Messages.add(perf-test, Performance tests without security for 1236 completed) + perf-test.writeFile({file=messages/perf-test.msg, text=Performance tests without security for 1236 completed}) + perf-test.stash({includes=messages/*, name=messages-perf-test}) + perf-test.script(groovy.lang.Closure) + perf-test.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-1.3.0-bundle.yml, jobName=perf-test, buildNumber=1236}) uploadTestResults.legacySCM(groovy.lang.Closure) uploadTestResults.library({identifier=jenkins@20211123, retriever=null}) uploadTestResults.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) @@ -107,5 +127,21 @@ Messages.add(test_stage, test://artifact.url/perf-test/1.3.0/1236/linux/x64/test-results/) uploadTestResults.writeFile({file=messages/test_stage.msg, text=test://artifact.url/perf-test/1.3.0/1236/linux/x64/test-results/}) uploadTestResults.stash({includes=messages/*, name=messages-test_stage}) - RunPerfTestScript_Jenkinsfile.postCleanup() + perf-test.postCleanup() postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) + perf-test.node(Jenkins-Agent-al2-x64-c54xlarge-Docker-Host, groovy.lang.Closure) + perf-test.script(groovy.lang.Closure) + Messages.asBoolean() + Messages.get([perf-test]) + perf-test.unstash({name=messages-perf-test}) + perf-test.findFiles({excludes=, glob=messages/*}) + perf-test.dir(messages, groovy.lang.Closure) + perf-test.deleteDir() + perf-test.publishNotification({icon=:white_check_mark:, message=Performance Tests Successful, extra=, credentialsId=INTEG_TEST_WEBHOOK}) + publishNotification.string({credentialsId=INTEG_TEST_WEBHOOK, variable=WEBHOOK_URL}) + publishNotification.withCredentials([WEBHOOK_URL], groovy.lang.Closure) + publishNotification.sh(curl -XPOST --header "Content-Type: application/json" --data '{"result_text":":white_check_mark: perf-test [1236] Performance Tests Successful +Build: test://artifact.url +"}' "WEBHOOK_URL") + perf-test.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt new file mode 100644 index 0000000000..938d295bea --- /dev/null +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt @@ -0,0 +1,91 @@ + perf-test.run() + perf-test.legacySCM(groovy.lang.Closure) + perf-test.library({identifier=jenkins@20211118, retriever=null}) + perf-test.pipeline(groovy.lang.Closure) + perf-test.timeout({time=10, unit=HOURS}) + perf-test.echo(Executing on agent [label:none]) + perf-test.stage(validate-and-set-parameters, groovy.lang.Closure) + perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) + perf-test.script(groovy.lang.Closure) + perf-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) + downloadBuildManifest.legacySCM(groovy.lang.Closure) + downloadBuildManifest.library({identifier=jenkins@20211123, retriever=null}) + downloadBuildManifest.sh(curl test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml) + downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) + BuildManifest.asBoolean() + BuildManifest.getArtifactBuildId() + BuildManifest.getArtifactArchitecture() + perf-test.echo(Skipping stage test-with-security) + perf-test.stage(test-without-security, groovy.lang.Closure) + perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) + perf-test.script(groovy.lang.Closure) + perf-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) + downloadBuildManifest.legacySCM(groovy.lang.Closure) + downloadBuildManifest.library({identifier=jenkins@20211123, retriever=null}) + downloadBuildManifest.sh(curl test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml) + downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) + BuildManifest.asBoolean() + perf-test.echo(BUNDLE_MANIFEST: tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml) + perf-test.echo(BUILD_ID: 1236) + perf-test.echo(Architecture: x64) + Messages.asBoolean() + Messages.add(perf-test, Performance tests for #1236) + perf-test.writeFile({file=messages/perf-test.msg, text=Performance tests for #1236}) + perf-test.stash({includes=messages/*, name=messages-perf-test}) + perf-test.runPerfTestScript({bundleManifest=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml, buildId=1236, architecture=x64, insecure=true, workload=nyc_taxis, testIterations=1, warmupIterations=1}) + runPerfTestScript.legacySCM(groovy.lang.Closure) + runPerfTestScript.library({identifier=jenkins@20211123, retriever=null}) + runPerfTestScript.readYaml({file=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) + BuildManifest.asBoolean() + runPerfTestScript.sh( + npm install -g fs-extra + npm install -g chalk@4.1.2 + npm install -g @aws-cdk/cloudformation-diff + npm install -g aws-cdk + npm install -g cdk-assume-role-credential-plugin@1.4.0 + ) + runPerfTestScript.sh( + pipenv install "dataclasses_json~=0.5" "aws_requests_auth~=0.4" "json2html~=1.3.0" + pipenv install "aws-cdk.core~=1.143.0" "aws_cdk.aws_ec2~=1.143.0" "aws_cdk.aws_iam~=1.143.0" + pipenv install "boto3~=1.18" "setuptools~=57.4" "retry~=0.9" + ) + runPerfTestScript.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + runPerfTestScript.s3Download({file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true}) + runPerfTestScript.sh(./test.sh perf-test --stack test-single-1236-x64 --bundle-manifest tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml --config config.yml --force-insecure-mode --workload nyc_taxis --test-iters 1 --warmup-iters 1) + Messages.asBoolean() + Messages.add(perf-test, Performance tests without security for 1236 completed) + perf-test.writeFile({file=messages/perf-test.msg, text=Performance tests without security for 1236 completed}) + perf-test.stash({includes=messages/*, name=messages-perf-test}) + perf-test.script(groovy.lang.Closure) + perf-test.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml, jobName=perf-test, buildNumber=1236}) + uploadTestResults.legacySCM(groovy.lang.Closure) + uploadTestResults.library({identifier=jenkins@20211123, retriever=null}) + uploadTestResults.readYaml({file=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) + BuildManifest.asBoolean() + BuildManifest.getArtifactRoot(perf-test, 1236) + uploadTestResults.echo(Uploading to s3://test_bucket/perf-test/1.3.0/1236/linux/x64) + uploadTestResults.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + uploadTestResults.s3Upload({file=test-results, bucket=test_bucket, path=perf-test/1.3.0/1236/linux/x64/test-results}) + BuildManifest.getArtifactRootUrl(test://artifact.url, perf-test, 1236) + Messages.asBoolean() + Messages.add(test_stage, test://artifact.url/perf-test/1.3.0/1236/linux/x64/test-results/) + uploadTestResults.writeFile({file=messages/test_stage.msg, text=test://artifact.url/perf-test/1.3.0/1236/linux/x64/test-results/}) + uploadTestResults.stash({includes=messages/*, name=messages-test_stage}) + perf-test.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) + perf-test.node(Jenkins-Agent-al2-x64-c54xlarge-Docker-Host, groovy.lang.Closure) + perf-test.script(groovy.lang.Closure) + Messages.asBoolean() + Messages.get([perf-test]) + perf-test.unstash({name=messages-perf-test}) + perf-test.findFiles({excludes=, glob=messages/*}) + perf-test.dir(messages, groovy.lang.Closure) + perf-test.deleteDir() + perf-test.publishNotification({icon=:white_check_mark:, message=Performance Tests Successful, extra=, credentialsId=INTEG_TEST_WEBHOOK}) + publishNotification.string({credentialsId=INTEG_TEST_WEBHOOK, variable=WEBHOOK_URL}) + publishNotification.withCredentials([WEBHOOK_URL], groovy.lang.Closure) + publishNotification.sh(curl -XPOST --header "Content-Type: application/json" --data '{"result_text":":white_check_mark: perf-test [1236] Performance Tests Successful +Build: test://artifact.url +"}' "WEBHOOK_URL") + perf-test.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) diff --git a/tests/jenkins/jobs/RunPerfTestScriptWithoutSecurity_Jenkinsfile b/tests/jenkins/jobs/RunPerfTestScriptWithoutSecurity_Jenkinsfile deleted file mode 100644 index 25629ee195..0000000000 --- a/tests/jenkins/jobs/RunPerfTestScriptWithoutSecurity_Jenkinsfile +++ /dev/null @@ -1,179 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -lib = library(identifier: "jenkins@20211118", retriever: legacySCM(scm)) - -pipeline { - agent none - options { - timeout(time: 10, unit: 'HOURS') - } - parameters { - string( - name: 'GITHUB_TOKEN', - description: 'Github token for account access.', - trim: true - ) - string( - name: 'BUNDLE_MANIFEST_URL', - description: 'The bundle manifest URL, e.g. https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.2.2/98/linux/x64/builds/opensearch/manifest.yml.', - trim: true - ) - string( - defaultValue: 'nyc_taxis', - name: 'TEST_WORKLOAD', - description: 'The workload name from OpenSearch Benchmark Workloads for Mensor (internal client).', - trim: true - ) - string( - defaultValue: '1', - name: 'TEST_ITERATIONS', - description: 'Number of times to run a workload for Mensor (internal client).', - trim: true - ) - string( - defaultValue: '0', - name: 'WARMUP_ITERATIONS', - description: 'Number of times to run a workload before collecting data for Mensor (internal client).', - trim: true - ) - } - - stages { - stage('validate-and-set-parameters') { - agent { - docker { - label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' - image 'opensearchstaging/ci-runner:ci-runner-centos7-v1' - alwaysPull true - } - } - steps { - script { - if (BUNDLE_MANIFEST_URL == '') { - currentBuild.result = 'ABORTED' - error("Performance Tests failed to start. Missing parameter: BUNDLE_MANIFEST_URL.") - } - if (TEST_ITERATIONS != null && !TEST_ITERATIONS.isInteger()) { - currentBuild.result = 'ABORTED' - error("Performance Tests failed to start. Invalid value for parameter: TEST_ITERATIONS. Value should be an integer.") - } - if (WARMUP_ITERATIONS != null && !WARMUP_ITERATIONS.isInteger()) { - currentBuild.result = 'ABORTED' - error("Performance Tests failed to start. Invalid value for parameter: WARMUP_ITERATIONS. Value should be an integer.") - } - echo "Security: ${HAS_SECURITY}" - } - } - } - stage('perf-test') { - parallel { - stage('test-with-security') { - agent { - docker { - label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' - image 'opensearchstaging/ci-runner:ci-runner-centos7-v1' - alwaysPull true - } - } - when { - environment name: 'HAS_SECURITY', value: 'true' - } - steps { - script { - def bundleManifestObj = downloadBuildManifest( - url: BUNDLE_MANIFEST_URL, - path: BUNDLE_MANIFEST - ) - echo "BUNDLE_MANIFEST: ${BUNDLE_MANIFEST}" - echo "BUILD_ID: ${BUILD_ID}" - echo "BUILD_ID: ${BUILD_ID}" - echo "Security: ${HAS_SECURITY}" - echo "Architecture: ${ARCHITECTURE}" - - runPerfTestScript(bundleManifest: "${BUNDLE_MANIFEST}", - buildId: "${BUILD_ID}", - architecture: "${ARCHITECTURE}", - insecure: false, - workload: TEST_WORKLOAD, - testIterations: TEST_ITERATIONS, - warmupIterations: WARMUP_ITERATIONS) - } - } - post { - success { - script { - uploadTestResults( - buildManifestFileName: "${BUNDLE_MANIFEST}", - jobName: 'perf-test', - buildNumber: "${BUILD_ID}" - ) - } - postCleanup() - } - failure { - postCleanup() - } - aborted { - postCleanup() - } - } - } - stage('test-without-security') { - agent { - docker { - label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' - image 'opensearchstaging/ci-runner:ci-runner-centos7-v1' - alwaysPull true - } - } - steps { - script { - def bundleManifestObj = downloadBuildManifest( - url: BUNDLE_MANIFEST_URL, - path: BUNDLE_MANIFEST - ) - - echo "BUNDLE_MANIFEST: ${BUNDLE_MANIFEST}" - echo "BUILD_ID: ${BUILD_ID}" - echo "BUILD_ID: ${BUILD_ID}" - echo "Architecture: ${ARCHITECTURE}" - - - runPerfTestScript(bundleManifest: "${BUNDLE_MANIFEST}", - buildId: "${BUILD_ID}", - architecture: "${ARCHITECTURE}", - insecure: true, - workload: TEST_WORKLOAD, - testIterations: TEST_ITERATIONS, - warmupIterations: WARMUP_ITERATIONS) - } - } - post { - success { - script { - uploadTestResults( - buildManifestFileName: "${BUNDLE_MANIFEST}", - jobName: 'perf-test', - buildNumber: "${BUILD_ID}" - ) - } - postCleanup() - } - failure { - postCleanup() - } - aborted { - postCleanup() - } - } - } - } - } - } -} \ No newline at end of file diff --git a/tests/jenkins/jobs/RunPerfTestScriptWithoutSecurity_Jenkinsfile.txt b/tests/jenkins/jobs/RunPerfTestScriptWithoutSecurity_Jenkinsfile.txt deleted file mode 100644 index 64b7bdbaa7..0000000000 --- a/tests/jenkins/jobs/RunPerfTestScriptWithoutSecurity_Jenkinsfile.txt +++ /dev/null @@ -1,61 +0,0 @@ - RunPerfTestScriptWithoutSecurity_Jenkinsfile.run() - RunPerfTestScriptWithoutSecurity_Jenkinsfile.legacySCM(groovy.lang.Closure) - RunPerfTestScriptWithoutSecurity_Jenkinsfile.library({identifier=jenkins@20211118, retriever=null}) - RunPerfTestScriptWithoutSecurity_Jenkinsfile.pipeline(groovy.lang.Closure) - RunPerfTestScriptWithoutSecurity_Jenkinsfile.timeout({time=10, unit=HOURS}) - RunPerfTestScriptWithoutSecurity_Jenkinsfile.echo(Executing on agent [label:none]) - RunPerfTestScriptWithoutSecurity_Jenkinsfile.stage(validate-and-set-parameters, groovy.lang.Closure) - RunPerfTestScriptWithoutSecurity_Jenkinsfile.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) - RunPerfTestScriptWithoutSecurity_Jenkinsfile.script(groovy.lang.Closure) - RunPerfTestScriptWithoutSecurity_Jenkinsfile.echo(Security: false) - RunPerfTestScriptWithoutSecurity_Jenkinsfile.echo(Skipping stage test-with-security) - RunPerfTestScriptWithoutSecurity_Jenkinsfile.stage(test-without-security, groovy.lang.Closure) - RunPerfTestScriptWithoutSecurity_Jenkinsfile.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) - RunPerfTestScriptWithoutSecurity_Jenkinsfile.script(groovy.lang.Closure) - RunPerfTestScriptWithoutSecurity_Jenkinsfile.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) - downloadBuildManifest.legacySCM(groovy.lang.Closure) - downloadBuildManifest.library({identifier=jenkins@20211123, retriever=null}) - downloadBuildManifest.sh(curl test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml) - downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) - BuildManifest.asBoolean() - RunPerfTestScriptWithoutSecurity_Jenkinsfile.echo(BUNDLE_MANIFEST: tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml) - RunPerfTestScriptWithoutSecurity_Jenkinsfile.echo(BUILD_ID: 1236) - RunPerfTestScriptWithoutSecurity_Jenkinsfile.echo(BUILD_ID: 1236) - RunPerfTestScriptWithoutSecurity_Jenkinsfile.echo(Architecture: x64) - RunPerfTestScriptWithoutSecurity_Jenkinsfile.runPerfTestScript({bundleManifest=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml, buildId=1236, architecture=x64, insecure=true, workload=nyc_taxis, testIterations=1, warmupIterations=1}) - runPerfTestScript.legacySCM(groovy.lang.Closure) - runPerfTestScript.library({identifier=jenkins@20211123, retriever=null}) - runPerfTestScript.readYaml({file=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) - BuildManifest.asBoolean() - runPerfTestScript.sh( - npm install -g fs-extra - npm install -g chalk@4.1.2 - npm install -g @aws-cdk/cloudformation-diff - npm install -g aws-cdk - npm install -g cdk-assume-role-credential-plugin@1.4.0 - ) - runPerfTestScript.sh( - pipenv install "dataclasses_json~=0.5" "aws_requests_auth~=0.4" "json2html~=1.3.0" - pipenv install "aws-cdk.core~=1.143.0" "aws_cdk.aws_ec2~=1.143.0" "aws_cdk.aws_iam~=1.143.0" - pipenv install "boto3~=1.18" "setuptools~=57.4" "retry~=0.9" - ) - runPerfTestScript.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - runPerfTestScript.s3Download({file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true}) - runPerfTestScript.sh(./test.sh perf-test --stack test-single-1236-x64 --bundle-manifest tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml --config config.yml --force-insecure-mode --workload nyc_taxis --test-iters 1 --warmup-iters 1) - RunPerfTestScriptWithoutSecurity_Jenkinsfile.script(groovy.lang.Closure) - RunPerfTestScriptWithoutSecurity_Jenkinsfile.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml, jobName=perf-test, buildNumber=1236}) - uploadTestResults.legacySCM(groovy.lang.Closure) - uploadTestResults.library({identifier=jenkins@20211123, retriever=null}) - uploadTestResults.readYaml({file=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) - BuildManifest.asBoolean() - BuildManifest.getArtifactRoot(perf-test, 1236) - uploadTestResults.echo(Uploading to s3://test_bucket/perf-test/1.3.0/1236/linux/x64) - uploadTestResults.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - uploadTestResults.s3Upload({file=test-results, bucket=test_bucket, path=perf-test/1.3.0/1236/linux/x64/test-results}) - BuildManifest.getArtifactRootUrl(test://artifact.url, perf-test, 1236) - Messages.asBoolean() - Messages.add(test_stage, test://artifact.url/perf-test/1.3.0/1236/linux/x64/test-results/) - uploadTestResults.writeFile({file=messages/test_stage.msg, text=test://artifact.url/perf-test/1.3.0/1236/linux/x64/test-results/}) - uploadTestResults.stash({includes=messages/*, name=messages-test_stage}) - RunPerfTestScriptWithoutSecurity_Jenkinsfile.postCleanup() - postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) diff --git a/tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile b/tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile deleted file mode 100644 index d44b8a37ac..0000000000 --- a/tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile +++ /dev/null @@ -1,176 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -lib = library(identifier: "jenkins@20211118", retriever: legacySCM(scm)) - -pipeline { - agent none - options { - timeout(time: 10, unit: 'HOURS') - } - parameters { - string( - name: 'GITHUB_TOKEN', - description: 'Github token for account access.', - trim: true - ) - string( - name: 'BUNDLE_MANIFEST_URL', - description: 'The bundle manifest URL, e.g. https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.2.2/98/linux/x64/builds/opensearch/manifest.yml.', - trim: true - ) - string( - defaultValue: 'nyc_taxis', - name: 'TEST_WORKLOAD', - description: 'The workload name from OpenSearch Benchmark Workloads for Mensor (internal client).', - trim: true - ) - string( - defaultValue: '1', - name: 'TEST_ITERATIONS', - description: 'Number of times to run a workload for Mensor (internal client).', - trim: true - ) - string( - defaultValue: '0', - name: 'WARMUP_ITERATIONS', - description: 'Number of times to run a workload before collecting data for Mensor (internal client).', - trim: true - ) - } - - stages { - stage('validate-and-set-parameters') { - agent { - docker { - label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' - image 'opensearchstaging/ci-runner:ci-runner-centos7-v1' - alwaysPull true - } - } - steps { - script { - if (BUNDLE_MANIFEST_URL == '') { - currentBuild.result = 'ABORTED' - error("Performance Tests failed to start. Missing parameter: BUNDLE_MANIFEST_URL.") - } - if (TEST_ITERATIONS != null && !TEST_ITERATIONS.isInteger()) { - currentBuild.result = 'ABORTED' - error("Performance Tests failed to start. Invalid value for parameter: TEST_ITERATIONS. Value should be an integer.") - } - if (WARMUP_ITERATIONS != null && !WARMUP_ITERATIONS.isInteger()) { - currentBuild.result = 'ABORTED' - error("Performance Tests failed to start. Invalid value for parameter: WARMUP_ITERATIONS. Value should be an integer.") - } - echo "Security: ${HAS_SECURITY}" - } - } - } - stage('perf-test') { - parallel { - stage('test-with-security') { - agent { - docker { - label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' - image 'opensearchstaging/ci-runner:ci-runner-centos7-v1' - alwaysPull true - } - } - steps { - script { - def bundleManifestObj = downloadBuildManifest( - url: BUNDLE_MANIFEST_URL, - path: BUNDLE_MANIFEST - ) - echo "BUNDLE_MANIFEST: ${BUNDLE_MANIFEST}" - echo "BUILD_ID: ${BUILD_ID}" - echo "BUILD_ID: ${BUILD_ID}" - echo "Security: ${HAS_SECURITY}" - echo "Architecture: ${ARCHITECTURE}" - - runPerfTestScript(bundleManifest: "${BUNDLE_MANIFEST}", - buildId: "${BUILD_ID}", - architecture: "${ARCHITECTURE}", - insecure: false, - workload: TEST_WORKLOAD, - testIterations: TEST_ITERATIONS, - warmupIterations: WARMUP_ITERATIONS) - } - } - post { - success { - script { - uploadTestResults( - buildManifestFileName: "${BUNDLE_MANIFEST}", - jobName: 'perf-test', - buildNumber: "${BUILD_ID}" - ) - } - postCleanup() - } - failure { - postCleanup() - } - aborted { - postCleanup() - } - } - } - stage('test-without-security') { - agent { - docker { - label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' - image 'opensearchstaging/ci-runner:ci-runner-centos7-v1' - alwaysPull true - } - } - steps { - script { - def bundleManifestObj = downloadBuildManifest( - url: BUNDLE_MANIFEST_URL, - path: BUNDLE_MANIFEST - ) - - echo "BUNDLE_MANIFEST: ${BUNDLE_MANIFEST}" - echo "BUILD_ID: ${BUILD_ID}" - echo "BUILD_ID: ${BUILD_ID}" - echo "Architecture: ${ARCHITECTURE}" - - - runPerfTestScript(bundleManifest: "${BUNDLE_MANIFEST}", - buildId: "${BUILD_ID}", - architecture: "${ARCHITECTURE}", - insecure: true, - workload: TEST_WORKLOAD, - testIterations: TEST_ITERATIONS, - warmupIterations: WARMUP_ITERATIONS) - } - } - post { - success { - script { - uploadTestResults( - buildManifestFileName: "${BUNDLE_MANIFEST}", - jobName: 'perf-test', - buildNumber: "${BUILD_ID}" - ) - } - postCleanup() - } - failure { - postCleanup() - } - aborted { - postCleanup() - } - } - } - } - } - } -} \ No newline at end of file diff --git a/tests/jenkins/lib-testers/RunPerfTestScriptLibTest.groovy b/tests/jenkins/lib-testers/RunPerfTestScriptLibTest.groovy index 498615c487..242eb34f9c 100644 --- a/tests/jenkins/lib-testers/RunPerfTestScriptLibTest.groovy +++ b/tests/jenkins/lib-testers/RunPerfTestScriptLibTest.groovy @@ -32,23 +32,33 @@ class RunPerfTestScriptLibTester extends LibFunctionTester { closure.delegate = delegate return helper.callClosure(closure) }) + helper.registerAllowedMethod('findFiles', [Map.class], null) + helper.registerAllowedMethod("withCredentials", [Map]) + helper.registerAllowedMethod("downloadBuildManifest", [Map], { + c -> lib.jenkins.BuildManifest.new(readYaml(file: bundleManifest)) + }) - binding.setVariable('GITHUB_TOKEN', 'test_token') + binding.setVariable('AGENT_LABEL', 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host') + binding.setVariable('AGENT_IMAGE', 'opensearchstaging/ci-runner:ci-runner-centos7-v1') + binding.setVariable('ARCHITECTURE', 'x64') binding.setVariable('ARTIFACT_BUCKET_NAME', 'test_bucket') - binding.setVariable('PUBLIC_ARTIFACT_URL', 'test://artifact.url') - binding.setVariable('STAGE_NAME', 'test_stage') - binding.setVariable('PERF_TEST_CONFIG_LOCATION', 'test_config') binding.setVariable('ARTIFACT_DOWNLOAD_ROLE_NAME', 'Dummy_Download_Role') binding.setVariable('AWS_ACCOUNT_PUBLIC', 'dummy_account') - - binding.setVariable('BUNDLE_MANIFEST_URL', 'test://artifact.url') - binding.setVariable('BUNDLE_MANIFEST', bundleManifest) binding.setVariable('BUILD_ID', buildId) + binding.setVariable('BUILD_NUMBER', buildId) + binding.setVariable('BUILD_URL', 'test://artifact.url') + binding.setVariable('BUNDLE_MANIFEST', bundleManifest) + binding.setVariable('BUNDLE_MANIFEST_URL', 'test://artifact.url') + binding.setVariable('GITHUB_TOKEN', 'test_token') binding.setVariable('HAS_SECURITY', security) + binding.setVariable('JOB_NAME', 'perf-test') + binding.setVariable('PERF_TEST_CONFIG_LOCATION', 'test_config') + binding.setVariable('PUBLIC_ARTIFACT_URL', 'test://artifact.url') + binding.setVariable('STAGE_NAME', 'test_stage') binding.setVariable('TEST_ITERATIONS', testIterations) - binding.setVariable('WARMUP_ITERATIONS', warmupIterations) binding.setVariable('TEST_WORKLOAD', workload) - binding.setVariable('ARCHITECTURE', 'x64') + binding.setVariable('WEBHOOK_URL', 'test://artifact.url') + binding.setVariable('WARMUP_ITERATIONS', warmupIterations) } From a19db1150ec62ff972a8d776b853b11a81568c6e Mon Sep 17 00:00:00 2001 From: Kunal Kotwani Date: Fri, 1 Apr 2022 16:12:53 -0700 Subject: [PATCH 10/12] Update secure mode flag name, pythonify branch code Signed-off-by: Kunal Kotwani --- src/run_perf_test.py | 15 ++++++++------- src/test_workflow/perf_test/perf_test_suite.py | 10 +--------- .../TestRunNonSecurityPerfTestScript.groovy | 4 ++-- tests/jenkins/TestRunPerfTestScript.groovy | 2 +- .../perf-test-with-security.jenkinsfile.txt | 2 +- .../opensearch/perf-test.jenkinsfile.txt | 2 +- tests/test_run_perf_test.py | 2 +- vars/runPerfTestScript.groovy | 2 +- 8 files changed, 16 insertions(+), 23 deletions(-) diff --git a/src/run_perf_test.py b/src/run_perf_test.py index caca751578..9be49ccce1 100644 --- a/src/run_perf_test.py +++ b/src/run_perf_test.py @@ -36,7 +36,7 @@ def main(): parser.add_argument("--bundle-manifest", type=argparse.FileType("r"), help="Bundle Manifest file.", required=True) parser.add_argument("--stack", dest="stack", help="Stack name for performance test") parser.add_argument("--config", type=argparse.FileType("r"), help="Config file.", required=True) - parser.add_argument("--force-insecure-mode", dest="insecure", action="store_true", + parser.add_argument("--without-security", dest="insecure", action="store_true", help="Force the security of the cluster to be disabled.", default=False) parser.add_argument("--keep", dest="keep", action="store_true", help="Do not delete the working temporary directory.") @@ -50,18 +50,19 @@ def main(): config = yaml.safe_load(args.config) security = "security" in manifest.components and not args.insecure - if security: - tests_dir = os.path.join(os.getcwd(), "test-results", "perf-test", "with-security") - else: - tests_dir = os.path.join(os.getcwd(), "test-results", "perf-test", "without-security") + tests_dir = os.path.join(os.getcwd(), "test-results", "perf-test", f"{'with' if security else 'without'}-security") os.makedirs(tests_dir, exist_ok=True) with TemporaryDirectory(keep=args.keep, chdir=True) as work_dir: current_workspace = os.path.join(work_dir.name, "infra") with GitRepository(get_infra_repo_url(), "main", current_workspace): with WorkingDirectory(current_workspace): - with PerfTestCluster.create(manifest, config, args.stack, security, current_workspace) \ - as (test_cluster_endpoint, test_cluster_port): + with PerfTestCluster.create( + manifest, + config, + args.stack, + security, + current_workspace) as (test_cluster_endpoint, test_cluster_port): perf_test_suite = PerfTestSuite(manifest, test_cluster_endpoint, security, current_workspace, tests_dir, args) retry_call(perf_test_suite.execute, tries=3, delay=60, backoff=2) diff --git a/src/test_workflow/perf_test/perf_test_suite.py b/src/test_workflow/perf_test/perf_test_suite.py index 3f6fc55ce8..d885690edd 100644 --- a/src/test_workflow/perf_test/perf_test_suite.py +++ b/src/test_workflow/perf_test/perf_test_suite.py @@ -17,19 +17,11 @@ def __init__(self, bundle_manifest, endpoint, security, current_workspace, test_ self.args = args self.command = ( f"pipenv run python test_config.py -i {self.endpoint} -b {self.manifest.build.id}" - f" -a {self.manifest.build.architecture} -p {self.current_workspace}" + f" -a {self.manifest.build.architecture} -p {os.getcwd() if test_results_path is None else test_results_path}" f" --workload {self.args.workload} --workload-options '{self.args.workload_options}'" f" --warmup-iters {self.args.warmup_iters} --test-iters {self.args.test_iters}" ) - if test_results_path is not None: - self.command = ( - f"pipenv run python test_config.py -i {self.endpoint} -b {self.manifest.build.id}" - f" -a {self.manifest.build.architecture} -p {test_results_path}" - f" --workload {self.args.workload} --workload-options '{self.args.workload_options}'" - f" --warmup-iters {self.args.warmup_iters} --test-iters {self.args.test_iters}" - ) - def execute(self): try: current_workspace = os.path.join(self.current_workspace, self.work_dir) diff --git a/tests/jenkins/TestRunNonSecurityPerfTestScript.groovy b/tests/jenkins/TestRunNonSecurityPerfTestScript.groovy index 387ac12076..79f153d575 100644 --- a/tests/jenkins/TestRunNonSecurityPerfTestScript.groovy +++ b/tests/jenkins/TestRunNonSecurityPerfTestScript.groovy @@ -15,7 +15,7 @@ import static org.hamcrest.CoreMatchers.equalTo import static org.hamcrest.CoreMatchers.hasItem import static org.hamcrest.MatcherAssert.assertThat -class TestRunSecurityPerfTestScript extends BuildPipelineTest { +class TestRunNonSecurityPerfTestScript extends BuildPipelineTest { @Before void setUp() { @@ -88,7 +88,7 @@ class TestRunSecurityPerfTestScript extends BuildPipelineTest { assertThat(testScriptCommands.size(), equalTo(1)) assertThat(testScriptCommands, hasItem( - "./test.sh perf-test --stack test-single-1236-x64 --bundle-manifest tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml --config config.yml --force-insecure-mode --workload nyc_taxis --test-iters 1 --warmup-iters 1".toString() + "./test.sh perf-test --stack test-single-1236-x64 --bundle-manifest tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml --config config.yml --without-security --workload nyc_taxis --test-iters 1 --warmup-iters 1".toString() )) def resultUploadScriptCommands = getCommandExecutions('s3Upload', 'test-results').findAll { diff --git a/tests/jenkins/TestRunPerfTestScript.groovy b/tests/jenkins/TestRunPerfTestScript.groovy index ab86b43630..5520ef1940 100644 --- a/tests/jenkins/TestRunPerfTestScript.groovy +++ b/tests/jenkins/TestRunPerfTestScript.groovy @@ -91,7 +91,7 @@ class TestRunPerfTestScript extends BuildPipelineTest { "./test.sh perf-test --stack test-single-security-1236-x64 --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config config.yml --workload nyc_taxis --test-iters 1 --warmup-iters 1".toString() )) assertThat(testScriptCommands, hasItem( - "./test.sh perf-test --stack test-single-1236-x64 --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config config.yml --force-insecure-mode --workload nyc_taxis --test-iters 1 --warmup-iters 1".toString() + "./test.sh perf-test --stack test-single-1236-x64 --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config config.yml --without-security --workload nyc_taxis --test-iters 1 --warmup-iters 1".toString() )) def resultUploadScriptCommands = getCommandExecutions('s3Upload', 'test-results').findAll { diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile.txt index 9aa9d84d95..f87e978372 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile.txt @@ -107,7 +107,7 @@ ) runPerfTestScript.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) runPerfTestScript.s3Download({file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true}) - runPerfTestScript.sh(./test.sh perf-test --stack test-single-1236-x64 --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config config.yml --force-insecure-mode --workload nyc_taxis --test-iters 1 --warmup-iters 1) + runPerfTestScript.sh(./test.sh perf-test --stack test-single-1236-x64 --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config config.yml --without-security --workload nyc_taxis --test-iters 1 --warmup-iters 1) Messages.asBoolean() Messages.add(perf-test, Performance tests without security for 1236 completed) perf-test.writeFile({file=messages/perf-test.msg, text=Performance tests without security for 1236 completed}) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt index 938d295bea..5d06d7bf9c 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt @@ -51,7 +51,7 @@ ) runPerfTestScript.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) runPerfTestScript.s3Download({file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true}) - runPerfTestScript.sh(./test.sh perf-test --stack test-single-1236-x64 --bundle-manifest tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml --config config.yml --force-insecure-mode --workload nyc_taxis --test-iters 1 --warmup-iters 1) + runPerfTestScript.sh(./test.sh perf-test --stack test-single-1236-x64 --bundle-manifest tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml --config config.yml --without-security --workload nyc_taxis --test-iters 1 --warmup-iters 1) Messages.asBoolean() Messages.add(perf-test, Performance tests without security for 1236 completed) perf-test.writeFile({file=messages/perf-test.msg, text=Performance tests without security for 1236 completed}) diff --git a/tests/test_run_perf_test.py b/tests/test_run_perf_test.py index 6dcb216435..d96f0063d9 100644 --- a/tests/test_run_perf_test.py +++ b/tests/test_run_perf_test.py @@ -67,7 +67,7 @@ def test_default_execute_perf_test(self, mock_git, mock_temp, mock_working_dir, self.assertIn(True, mock_suite.call_args[0]) @patch("argparse._sys.argv", ["run_perf_test.py", "--bundle-manifest", OPENSEARCH_BUNDLE_MANIFEST, - "--stack", "test-stack", "--config", PERF_TEST_CONFIG, "--force-insecure-mode"]) + "--stack", "test-stack", "--config", PERF_TEST_CONFIG, "--without-security"]) @patch("run_perf_test.PerfTestCluster.create") @patch("run_perf_test.PerfTestSuite") @patch("run_perf_test.WorkingDirectory") diff --git a/vars/runPerfTestScript.groovy b/vars/runPerfTestScript.groovy index c90f03296e..bec1267c2c 100644 --- a/vars/runPerfTestScript.groovy +++ b/vars/runPerfTestScript.groovy @@ -15,7 +15,7 @@ void call(Map args = [:]) { "--stack test-single-security-${args.buildId}-${args.architecture}", "--bundle-manifest ${args.bundleManifest}", "--config config.yml", - args.insecure ? "--force-insecure-mode" : "", + args.insecure ? "--without-security" : "", isNullOrEmpty(args.workload) ? "" : "--workload ${args.workload}", isNullOrEmpty(args.testIterations) ? "" : "--test-iters ${args.testIterations}", isNullOrEmpty(args.warmupIterations) ? "" : "--warmup-iters ${args.warmupIterations}", From fad13dbb98c0c8f3f2767fd5710aa88d74bbf770 Mon Sep 17 00:00:00 2001 From: Kunal Kotwani Date: Mon, 4 Apr 2022 09:10:29 -0700 Subject: [PATCH 11/12] Fix notifications for perf-tests Signed-off-by: Kunal Kotwani --- jenkins/opensearch/perf-test.jenkinsfile | 47 ++++++++++--------- .../perf-test-with-security.jenkinsfile.txt | 37 ++++++++++----- .../opensearch/perf-test.jenkinsfile.txt | 21 ++++++--- 3 files changed, 64 insertions(+), 41 deletions(-) diff --git a/jenkins/opensearch/perf-test.jenkinsfile b/jenkins/opensearch/perf-test.jenkinsfile index fa96a95208..bcd97905e6 100644 --- a/jenkins/opensearch/perf-test.jenkinsfile +++ b/jenkins/opensearch/perf-test.jenkinsfile @@ -9,6 +9,7 @@ pipeline { AGENT_LABEL = 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' AGENT_IMAGE = 'opensearchstaging/ci-runner:ci-runner-centos7-v1' BUNDLE_MANIFEST = 'bundle-manifest.yml' + JOB_NAME = 'perf-test' } parameters { string( @@ -76,6 +77,8 @@ pipeline { env.BUILD_ID = buildId env.HAS_SECURITY = bundleManifestObj.components.containsKey("security") env.ARCHITECTURE = bundleManifestObj.getArtifactArchitecture() + echo "HAS_SECURITY: ${env.HAS_SECURITY}" + lib.jenkins.Messages.new(this).add(JOB_NAME, "Performance tests for #${BUILD_ID}") } } } @@ -90,7 +93,7 @@ pipeline { } } when { - expression { HAS_SECURITY == true } + expression { return env.HAS_SECURITY } } steps { script { @@ -102,26 +105,26 @@ pipeline { echo "BUILD_ID: ${BUILD_ID}" echo "Architecture: ${ARCHITECTURE}" - lib.jenkins.Messages.new(this).add('perf-test', "Performance tests for #${BUILD_ID}") - - runPerfTestScript(bundleManifest: "${BUNDLE_MANIFEST}", - buildId: "${BUILD_ID}", - architecture: "${ARCHITECTURE}", + runPerfTestScript(bundleManifest: BUNDLE_MANIFEST, + buildId: BUILD_ID, + architecture: ARCHITECTURE, insecure: false, workload: TEST_WORKLOAD, testIterations: TEST_ITERATIONS, warmupIterations: WARMUP_ITERATIONS) - lib.jenkins.Messages.new(this).add('perf-test', "Performance tests with security for ${BUILD_ID} completed") + lib.jenkins.Messages.new(this).add(JOB_NAME, + lib.jenkins.Messages.new(this).get([JOB_NAME]) + + "\nPerformance tests with security for ${BUILD_ID} completed") } } post { success { script { uploadTestResults( - buildManifestFileName: "${BUNDLE_MANIFEST}", - jobName: 'perf-test', - buildNumber: "${BUILD_ID}" + buildManifestFileName: BUNDLE_MANIFEST, + jobName: JOB_NAME, + buildNumber: BUILD_ID ) } postCleanup() @@ -153,26 +156,26 @@ pipeline { echo "BUILD_ID: ${BUILD_ID}" echo "Architecture: ${ARCHITECTURE}" - lib.jenkins.Messages.new(this).add('perf-test', "Performance tests for #${BUILD_ID}") - - runPerfTestScript(bundleManifest: "${BUNDLE_MANIFEST}", - buildId: "${BUILD_ID}", - architecture: "${ARCHITECTURE}", + runPerfTestScript(bundleManifest: BUNDLE_MANIFEST, + buildId: BUILD_ID, + architecture: ARCHITECTURE, insecure: true, workload: TEST_WORKLOAD, testIterations: TEST_ITERATIONS, warmupIterations: WARMUP_ITERATIONS) - lib.jenkins.Messages.new(this).add('perf-test', "Performance tests without security for ${BUILD_ID} completed") + lib.jenkins.Messages.new(this).add(JOB_NAME, + lib.jenkins.Messages.new(this).get([JOB_NAME]) + + "\nPerformance tests without security for ${BUILD_ID} completed") } } post { success { script { uploadTestResults( - buildManifestFileName: "${BUNDLE_MANIFEST}", - jobName: 'perf-test', - buildNumber: "${BUILD_ID}" + buildManifestFileName: BUNDLE_MANIFEST, + jobName: JOB_NAME, + buildNumber: BUILD_ID ) } postCleanup() @@ -193,14 +196,13 @@ pipeline { success { node(AGENT_LABEL) { script { - def stashed = lib.jenkins.Messages.new(this).get(['perf-test']) + def stashed = lib.jenkins.Messages.new(this).get([JOB_NAME]) publishNotification( icon: ':white_check_mark:', message: 'Performance Tests Successful', extra: stashed, credentialsId: 'INTEG_TEST_WEBHOOK', ) - postCleanup() } } @@ -208,14 +210,13 @@ pipeline { failure { node(AGENT_LABEL) { script { - def stashed = lib.jenkins.Messages.new(this).get(['perf-test']) + def stashed = lib.jenkins.Messages.new(this).get([JOB_NAME]) publishNotification( icon: ':warning:', message: 'Failed Performance Tests', extra: stashed, credentialsId: 'INTEG_TEST_WEBHOOK', ) - postCleanup() } } diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile.txt index f87e978372..571c7ac115 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile.txt @@ -15,6 +15,11 @@ BuildManifest.asBoolean() BuildManifest.getArtifactBuildId() BuildManifest.getArtifactArchitecture() + perf-test.echo(HAS_SECURITY: true) + Messages.asBoolean() + Messages.add(perf-test, Performance tests for #1236) + perf-test.writeFile({file=messages/perf-test.msg, text=Performance tests for #1236}) + perf-test.stash({includes=messages/*, name=messages-perf-test}) perf-test.stage(test-with-security, groovy.lang.Closure) perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) perf-test.script(groovy.lang.Closure) @@ -27,10 +32,6 @@ perf-test.echo(BUNDLE_MANIFEST: tests/jenkins/data/opensearch-1.3.0-bundle.yml) perf-test.echo(BUILD_ID: 1236) perf-test.echo(Architecture: x64) - Messages.asBoolean() - Messages.add(perf-test, Performance tests for #1236) - perf-test.writeFile({file=messages/perf-test.msg, text=Performance tests for #1236}) - perf-test.stash({includes=messages/*, name=messages-perf-test}) perf-test.runPerfTestScript({bundleManifest=tests/jenkins/data/opensearch-1.3.0-bundle.yml, buildId=1236, architecture=x64, insecure=false, workload=nyc_taxis, testIterations=1, warmupIterations=1}) runPerfTestScript.legacySCM(groovy.lang.Closure) runPerfTestScript.library({identifier=jenkins@20211123, retriever=null}) @@ -52,8 +53,16 @@ runPerfTestScript.s3Download({file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true}) runPerfTestScript.sh(./test.sh perf-test --stack test-single-security-1236-x64 --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config config.yml --workload nyc_taxis --test-iters 1 --warmup-iters 1) Messages.asBoolean() - Messages.add(perf-test, Performance tests with security for 1236 completed) - perf-test.writeFile({file=messages/perf-test.msg, text=Performance tests with security for 1236 completed}) + Messages.asBoolean() + Messages.get([perf-test]) + perf-test.unstash({name=messages-perf-test}) + perf-test.findFiles({excludes=, glob=messages/*}) + perf-test.dir(messages, groovy.lang.Closure) + perf-test.deleteDir() + Messages.add(perf-test, +Performance tests with security for 1236 completed) + perf-test.writeFile({file=messages/perf-test.msg, text= +Performance tests with security for 1236 completed}) perf-test.stash({includes=messages/*, name=messages-perf-test}) perf-test.script(groovy.lang.Closure) perf-test.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-1.3.0-bundle.yml, jobName=perf-test, buildNumber=1236}) @@ -84,10 +93,6 @@ perf-test.echo(BUNDLE_MANIFEST: tests/jenkins/data/opensearch-1.3.0-bundle.yml) perf-test.echo(BUILD_ID: 1236) perf-test.echo(Architecture: x64) - Messages.asBoolean() - Messages.add(perf-test, Performance tests for #1236) - perf-test.writeFile({file=messages/perf-test.msg, text=Performance tests for #1236}) - perf-test.stash({includes=messages/*, name=messages-perf-test}) perf-test.runPerfTestScript({bundleManifest=tests/jenkins/data/opensearch-1.3.0-bundle.yml, buildId=1236, architecture=x64, insecure=true, workload=nyc_taxis, testIterations=1, warmupIterations=1}) runPerfTestScript.legacySCM(groovy.lang.Closure) runPerfTestScript.library({identifier=jenkins@20211123, retriever=null}) @@ -109,8 +114,16 @@ runPerfTestScript.s3Download({file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true}) runPerfTestScript.sh(./test.sh perf-test --stack test-single-1236-x64 --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config config.yml --without-security --workload nyc_taxis --test-iters 1 --warmup-iters 1) Messages.asBoolean() - Messages.add(perf-test, Performance tests without security for 1236 completed) - perf-test.writeFile({file=messages/perf-test.msg, text=Performance tests without security for 1236 completed}) + Messages.asBoolean() + Messages.get([perf-test]) + perf-test.unstash({name=messages-perf-test}) + perf-test.findFiles({excludes=, glob=messages/*}) + perf-test.dir(messages, groovy.lang.Closure) + perf-test.deleteDir() + Messages.add(perf-test, +Performance tests without security for 1236 completed) + perf-test.writeFile({file=messages/perf-test.msg, text= +Performance tests without security for 1236 completed}) perf-test.stash({includes=messages/*, name=messages-perf-test}) perf-test.script(groovy.lang.Closure) perf-test.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-1.3.0-bundle.yml, jobName=perf-test, buildNumber=1236}) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt index 5d06d7bf9c..4e364c9add 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt @@ -15,6 +15,11 @@ BuildManifest.asBoolean() BuildManifest.getArtifactBuildId() BuildManifest.getArtifactArchitecture() + perf-test.echo(HAS_SECURITY: false) + Messages.asBoolean() + Messages.add(perf-test, Performance tests for #1236) + perf-test.writeFile({file=messages/perf-test.msg, text=Performance tests for #1236}) + perf-test.stash({includes=messages/*, name=messages-perf-test}) perf-test.echo(Skipping stage test-with-security) perf-test.stage(test-without-security, groovy.lang.Closure) perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) @@ -28,10 +33,6 @@ perf-test.echo(BUNDLE_MANIFEST: tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml) perf-test.echo(BUILD_ID: 1236) perf-test.echo(Architecture: x64) - Messages.asBoolean() - Messages.add(perf-test, Performance tests for #1236) - perf-test.writeFile({file=messages/perf-test.msg, text=Performance tests for #1236}) - perf-test.stash({includes=messages/*, name=messages-perf-test}) perf-test.runPerfTestScript({bundleManifest=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml, buildId=1236, architecture=x64, insecure=true, workload=nyc_taxis, testIterations=1, warmupIterations=1}) runPerfTestScript.legacySCM(groovy.lang.Closure) runPerfTestScript.library({identifier=jenkins@20211123, retriever=null}) @@ -53,8 +54,16 @@ runPerfTestScript.s3Download({file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true}) runPerfTestScript.sh(./test.sh perf-test --stack test-single-1236-x64 --bundle-manifest tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml --config config.yml --without-security --workload nyc_taxis --test-iters 1 --warmup-iters 1) Messages.asBoolean() - Messages.add(perf-test, Performance tests without security for 1236 completed) - perf-test.writeFile({file=messages/perf-test.msg, text=Performance tests without security for 1236 completed}) + Messages.asBoolean() + Messages.get([perf-test]) + perf-test.unstash({name=messages-perf-test}) + perf-test.findFiles({excludes=, glob=messages/*}) + perf-test.dir(messages, groovy.lang.Closure) + perf-test.deleteDir() + Messages.add(perf-test, +Performance tests without security for 1236 completed) + perf-test.writeFile({file=messages/perf-test.msg, text= +Performance tests without security for 1236 completed}) perf-test.stash({includes=messages/*, name=messages-perf-test}) perf-test.script(groovy.lang.Closure) perf-test.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml, jobName=perf-test, buildNumber=1236}) From a2a40d5533edfab8c231c56bf3292bea620cee57 Mon Sep 17 00:00:00 2001 From: Kunal Kotwani Date: Mon, 4 Apr 2022 09:22:48 -0700 Subject: [PATCH 12/12] Update perf-test jenkins call stack Signed-off-by: Kunal Kotwani --- .../opensearch/perf-test-with-security.jenkinsfile.txt | 8 ++++++-- .../opensearch/perf-test.jenkinsfile.txt | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile.txt index 571c7ac115..cfed0d2dd0 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile.txt @@ -153,8 +153,12 @@ Performance tests without security for 1236 completed}) perf-test.publishNotification({icon=:white_check_mark:, message=Performance Tests Successful, extra=, credentialsId=INTEG_TEST_WEBHOOK}) publishNotification.string({credentialsId=INTEG_TEST_WEBHOOK, variable=WEBHOOK_URL}) publishNotification.withCredentials([WEBHOOK_URL], groovy.lang.Closure) - publishNotification.sh(curl -XPOST --header "Content-Type: application/json" --data '{"result_text":":white_check_mark: perf-test [1236] Performance Tests Successful -Build: test://artifact.url + publishNotification.sh(curl -XPOST --header "Content-Type: application/json" --data '{"result_text":":white_check_mark: +JOB_NAME=perf-test +BUILD_NUMBER=[1236] +MESSAGE=Performance Tests Successful +BUILD_URL: test://artifact.url +MANIFEST: null "}' "WEBHOOK_URL") perf-test.postCleanup() postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt index 4e364c9add..e426dca7cd 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt @@ -93,8 +93,12 @@ Performance tests without security for 1236 completed}) perf-test.publishNotification({icon=:white_check_mark:, message=Performance Tests Successful, extra=, credentialsId=INTEG_TEST_WEBHOOK}) publishNotification.string({credentialsId=INTEG_TEST_WEBHOOK, variable=WEBHOOK_URL}) publishNotification.withCredentials([WEBHOOK_URL], groovy.lang.Closure) - publishNotification.sh(curl -XPOST --header "Content-Type: application/json" --data '{"result_text":":white_check_mark: perf-test [1236] Performance Tests Successful -Build: test://artifact.url + publishNotification.sh(curl -XPOST --header "Content-Type: application/json" --data '{"result_text":":white_check_mark: +JOB_NAME=perf-test +BUILD_NUMBER=[1236] +MESSAGE=Performance Tests Successful +BUILD_URL: test://artifact.url +MANIFEST: null "}' "WEBHOOK_URL") perf-test.postCleanup() postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true})