diff --git a/Pipfile b/Pipfile index df9cc4c762..4bf457ba7a 100644 --- a/Pipfile +++ b/Pipfile @@ -16,6 +16,7 @@ pytest = "*" coverage = "~=4.5.4" pytest-cov = "~=2.10.0" jproperties = "~=2.1.1" +retry = "~=0.9" sortedcontainers = "*" cerberus = "~=1.3.4" psutil = "~=5.8" diff --git a/Pipfile.lock b/Pipfile.lock index ee2655bc5a..f5e633f5ee 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "26c25166b1c1c35d0c8871229a8fb34baa37389ba0d5dd40633ac33c8d16e10a" + "sha256": "3da3cb40cf6d9fb381ad4499ae93f01db922e93f1bed2fd233bb880b2ad422cd" }, "pipfile-spec": 6, "requires": { @@ -56,19 +56,19 @@ }, "charset-normalizer": { "hashes": [ - "sha256:876d180e9d7432c5d1dfd4c5d26b72f099d503e8fcc0feb7532c9289be60fcbd", - "sha256:cb957888737fc0bbcd78e3df769addb41fd1ff8cf950dc9e7ad7793f1bf44455" + "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597", + "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df" ], "markers": "python_version >= '3'", - "version": "==2.0.10" + "version": "==2.0.12" }, "click": { "hashes": [ - "sha256:353f466495adaeb40b6b5f592f9f91cb22372351c84caeb068132442a4518ef3", - "sha256:410e932b050f5eed773c4cda94de75971c89cdb3155a72a0831139a79e5ecb5b" + "sha256:6a7a62563bbfabfda3a38f3023a1db4a35978c0abd76f6c9605ecd6554d6d9b1", + "sha256:8458d7b1287c5fb128c90e23381cf99dcde74beaf6c7ff6384ce84d6fe090adb" ], "markers": "python_version >= '3.6'", - "version": "==8.0.3" + "version": "==8.0.4" }, "coverage": { "hashes": [ @@ -125,18 +125,19 @@ }, "distro": { "hashes": [ - "sha256:83f5e5a09f9c5f68f60173de572930effbcc0287bb84fdc4426cb4168c088424", - "sha256:c8713330ab31a034623a9515663ed87696700b55f04556b97c39cd261aa70dc7" + "sha256:151aeccf60c216402932b52e40ee477a939f8d58898927378a02abbe852c1c39", + "sha256:d596311d707e692c2160c37807f83e3820c5d539d5a83e87cfb6babd8ba3a06b" ], - "version": "==1.6.0" + "markers": "python_version >= '3.6'", + "version": "==1.7.0" }, "filelock": { "hashes": [ - "sha256:38b4f4c989f9d06d44524df1b24bd19e167d851f19b50bf3e3559952dddc5b80", - "sha256:cf0fc6a2f8d26bd900f19bf33915ca70ba4dd8c56903eeb14e1e7a2fd7590146" + "sha256:9cd540a9352e432c7246a48fe4e8712b10acb1df2ad1f30e8c070b82ae1fed85", + "sha256:f8314284bfffbdcfa0ff3d7992b023d4c628ced6feb957351d4c48d059f56bc0" ], "markers": "python_version >= '3.7'", - "version": "==3.4.2" + "version": "==3.6.0" }, "flake8": { "hashes": [ @@ -148,11 +149,11 @@ }, "identify": { "hashes": [ - "sha256:6b4b5031f69c48bf93a646b90de9b381c6b5f560df4cbe0ed3cf7650ae741e4d", - "sha256:aa68609c7454dbcaae60a01ff6b8df1de9b39fe6e50b1f6107ec81dcda624aa6" + "sha256:3f3244a559290e7d3deb9e9adc7b33594c1bc85a9dd82e0f1be519bf12a1ec17", + "sha256:5f06b14366bd1facb88b00540a1de05b69b310cbc2654db3c7e07fa3a4339323" ], - "markers": "python_full_version >= '3.6.1'", - "version": "==2.4.4" + "markers": "python_version >= '3.7'", + "version": "==2.4.12" }, "idna": { "hashes": [ @@ -164,11 +165,11 @@ }, "importlib-metadata": { "hashes": [ - "sha256:92a8b58ce734b2a4494878e0ecf7d79ccd7a128b5fc6014c401e0b61f006f0f6", - "sha256:b7cf7d3fef75f1e4c80a96ca660efbd51473d7e8f39b5ab9210febc7809012a4" + "sha256:1208431ca90a8cca1a6b8af391bb53c1a2db74e5d1cef6ddced95d4b2062edc6", + "sha256:ea4c597ebf37142f827b8f39299579e31685c31d3a438b59f469406afd0f2539" ], "markers": "python_version < '3.8'", - "version": "==4.10.0" + "version": "==4.11.3" }, "iniconfig": { "hashes": [ @@ -202,29 +203,32 @@ }, "mypy": { "hashes": [ - "sha256:0038b21890867793581e4cb0d810829f5fd4441aa75796b53033af3aa30430ce", - "sha256:1171f2e0859cfff2d366da2c7092b06130f232c636a3f7301e3feb8b41f6377d", - "sha256:1b06268df7eb53a8feea99cbfff77a6e2b205e70bf31743e786678ef87ee8069", - "sha256:1b65714dc296a7991000b6ee59a35b3f550e0073411ac9d3202f6516621ba66c", - "sha256:1bf752559797c897cdd2c65f7b60c2b6969ffe458417b8d947b8340cc9cec08d", - "sha256:300717a07ad09525401a508ef5d105e6b56646f7942eb92715a1c8d610149714", - "sha256:3c5b42d0815e15518b1f0990cff7a705805961613e701db60387e6fb663fe78a", - "sha256:4365c60266b95a3f216a3047f1d8e3f895da6c7402e9e1ddfab96393122cc58d", - "sha256:50c7346a46dc76a4ed88f3277d4959de8a2bd0a0fa47fa87a4cde36fe247ac05", - "sha256:5b56154f8c09427bae082b32275a21f500b24d93c88d69a5e82f3978018a0266", - "sha256:74f7eccbfd436abe9c352ad9fb65872cc0f1f0a868e9d9c44db0893440f0c697", - "sha256:7b3f6f557ba4afc7f2ce6d3215d5db279bcf120b3cfd0add20a5d4f4abdae5bc", - "sha256:8c11003aaeaf7cc2d0f1bc101c1cc9454ec4cc9cb825aef3cafff8a5fdf4c799", - "sha256:8ca7f8c4b1584d63c9a0f827c37ba7a47226c19a23a753d52e5b5eddb201afcd", - "sha256:c89702cac5b302f0c5d33b172d2b55b5df2bede3344a2fbed99ff96bddb2cf00", - "sha256:d8f1ff62f7a879c9fe5917b3f9eb93a79b78aad47b533911b853a757223f72e7", - "sha256:d9d2b84b2007cea426e327d2483238f040c49405a6bf4074f605f0156c91a47a", - "sha256:e839191b8da5b4e5d805f940537efcaa13ea5dd98418f06dc585d2891d228cf0", - "sha256:f9fe20d0872b26c4bba1c1be02c5340de1019530302cf2dcc85c7f9fc3252ae0", - "sha256:ff3bf387c14c805ab1388185dd22d6b210824e164d4bb324b195ff34e322d166" + "sha256:080097eee5393fd740f32c63f9343580aaa0fb1cda0128fd859dfcf081321c3d", + "sha256:0d3bcbe146247997e03bf030122000998b076b3ac6925b0b6563f46d1ce39b50", + "sha256:0dd441fbacf48e19dc0c5c42fafa72b8e1a0ba0a39309c1af9c84b9397d9b15a", + "sha256:108f3c7e14a038cf097d2444fa0155462362c6316e3ecb2d70f6dd99cd36084d", + "sha256:3bada0cf7b6965627954b3a128903a87cac79a79ccd83b6104912e723ef16c7b", + "sha256:3cf77f138efb31727ee7197bc824c9d6d7039204ed96756cc0f9ca7d8e8fc2a4", + "sha256:42c216a33d2bdba08098acaf5bae65b0c8196afeb535ef4b870919a788a27259", + "sha256:465a6ce9ca6268cadfbc27a2a94ddf0412568a6b27640ced229270be4f5d394d", + "sha256:6a8e1f63357851444940351e98fb3252956a15f2cabe3d698316d7a2d1f1f896", + "sha256:745071762f32f65e77de6df699366d707fad6c132a660d1342077cbf671ef589", + "sha256:818cfc51c25a5dbfd0705f3ac1919fff6971eb0c02e6f1a1f6a017a42405a7c0", + "sha256:8e5974583a77d630a5868eee18f85ac3093caf76e018c510aeb802b9973304ce", + "sha256:8eaf55fdf99242a1c8c792247c455565447353914023878beadb79600aac4a2a", + "sha256:98f61aad0bb54f797b17da5b82f419e6ce214de0aa7e92211ebee9e40eb04276", + "sha256:b2ce2788df0c066c2ff4ba7190fa84f18937527c477247e926abeb9b1168b8cc", + "sha256:b30d29251dff4c59b2e5a1fa1bab91ff3e117b4658cb90f76d97702b7a2ae699", + "sha256:bf446223b2e0e4f0a4792938e8d885e8a896834aded5f51be5c3c69566495540", + "sha256:cbcc691d8b507d54cb2b8521f0a2a3d4daa477f62fe77f0abba41e5febb377b7", + "sha256:d051ce0946521eba48e19b25f27f98e5ce4dbc91fff296de76240c46b4464df0", + "sha256:d61b73c01fc1de799226963f2639af831307fe1556b04b7c25e2b6c267a3bc76", + "sha256:eea10982b798ff0ccc3b9e7e42628f932f552c5845066970e67cd6858655d52c", + "sha256:f79137d012ff3227866222049af534f25354c07a0d6b9a171dba9f1d6a1fdef4", + "sha256:fc5ecff5a3bbfbe20091b1cad82815507f5ae9c380a3a9bf40f740c70ce30a9b" ], "index": "pypi", - "version": "==0.931" + "version": "==0.941" }, "mypy-extensions": { "hashes": [ @@ -257,11 +261,11 @@ }, "platformdirs": { "hashes": [ - "sha256:1d7385c7db91728b83efd0ca99a5afb296cab9d0ed8313a45ed8ba17967ecfca", - "sha256:440633ddfebcc36264232365d7840a970e75e1018d15b4327d11f91909045fda" + "sha256:7535e70dfa32e84d4b34996ea99c5e432fa29a708d0f4e394bbcb2a8faa4f16d", + "sha256:bcae7cab893c2d310a711b70b24efb93334febe65f8de776ee320b517471e227" ], "markers": "python_version >= '3.7'", - "version": "==2.4.1" + "version": "==2.5.1" }, "pluggy": { "hashes": [ @@ -343,19 +347,19 @@ }, "pyparsing": { "hashes": [ - "sha256:04ff808a5b90911829c55c4e26f75fa5ca8a2f5f36aa3a51f68e27033341d3e4", - "sha256:d9bdec0013ef1eb5a84ab39a3b3868911598afa494f5faa038647101504e2b81" + "sha256:18ee9022775d270c55187733956460083db60b37d0d0fb357445f3094eed3eea", + "sha256:a6c06a88f252e6c322f65faf8f418b16213b51bdfaece0524c1c1bc30c63c484" ], "markers": "python_version >= '3.6'", - "version": "==3.0.6" + "version": "==3.0.7" }, "pytest": { "hashes": [ - "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89", - "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134" + "sha256:841132caef6b1ad17a9afde46dc4f6cfa59a05f9555aae5151f73bdf2820ca63", + "sha256:92f723789a8fdd7180b6b06483874feca4c48a5c76968e03bb3e7f806a1869ea" ], "index": "pypi", - "version": "==6.2.5" + "version": "==7.1.1" }, "pytest-cov": { "hashes": [ @@ -408,6 +412,14 @@ "index": "pypi", "version": "==2.27.1" }, + "retry": { + "hashes": [ + "sha256:ccddf89761fa2c726ab29391837d4327f819ea14d244c232a1d24c67a2f98606", + "sha256:f8bfa8b99b69c4506d6f5bd3b0aabf77f98cdb17f3c9fc3f5ca820033336fba4" + ], + "index": "pypi", + "version": "==0.9.2" + }, "ruyaml": { "hashes": [ "sha256:50e0ee3389c77ad340e209472e0effd41ae0275246df00cdad0a067532171755", @@ -416,6 +428,14 @@ "markers": "python_version >= '3.6'", "version": "==0.91.0" }, + "setuptools": { + "hashes": [ + "sha256:6599055eeb23bfef457d5605d33a4d68804266e6cb430b0fb12417c5efeae36c", + "sha256:782ef48d58982ddb49920c11a0c5c9c0b02e7d7d1c2ad0aa44e1a1e133051c96" + ], + "markers": "python_version >= '3.7'", + "version": "==60.10.0" + }, "six": { "hashes": [ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", @@ -442,36 +462,41 @@ }, "tomli": { "hashes": [ - "sha256:b5bde28da1fed24b9bd1d4d2b8cba62300bfb4ec9a6187a957e8ddb9434c5224", - "sha256:c292c34f58502a1eb2bbb9f5bbc9a5ebc37bee10ffb8c2d6bbdfa8eb13cc14e1" + "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", + "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f" ], "markers": "python_version >= '3.7'", - "version": "==2.0.0" + "version": "==2.0.1" }, "typed-ast": { "hashes": [ - "sha256:24058827d8f5d633f97223f5148a7d22628099a3d2efe06654ce872f46f07cdb", - "sha256:256115a5bc7ea9e665c6314ed6671ee2c08ca380f9d5f130bd4d2c1f5848d695", - "sha256:38cf5c642fa808300bae1281460d4f9b7617cf864d4e383054a5ef336e344d32", - "sha256:484137cab8ecf47e137260daa20bafbba5f4e3ec7fda1c1e69ab299b75fa81c5", - "sha256:4f30a2bcd8e68adbb791ce1567fdb897357506f7ea6716f6bbdd3053ac4d9471", - "sha256:591bc04e507595887160ed7aa8d6785867fb86c5793911be79ccede61ae96f4d", - "sha256:5b6ab14c56bc9c7e3c30228a0a0b54b915b1579613f6e463ba6f4eb1382e7fd4", - "sha256:5d8314c92414ce7481eee7ad42b353943679cf6f30237b5ecbf7d835519e1212", - "sha256:71dcda943a471d826ea930dd449ac7e76db7be778fcd722deb63642bab32ea3f", - "sha256:7c42707ab981b6cf4b73490c16e9d17fcd5227039720ca14abe415d39a173a30", - "sha256:9caaf2b440efb39ecbc45e2fabde809cbe56272719131a6318fd9bf08b58e2cb", - "sha256:a2b8d7007f6280e36fa42652df47087ac7b0a7d7f09f9468f07792ba646aac2d", - "sha256:a6d495c1ef572519a7bac9534dbf6d94c40e5b6a608ef41136133377bba4aa08", - "sha256:a80d84f535642420dd17e16ae25bb46c7f4c16ee231105e7f3eb43976a89670a", - "sha256:b53ae5de5500529c76225d18eeb060efbcec90ad5e030713fe8dab0fb4531631", - "sha256:b6d17f37f6edd879141e64a5db17b67488cfeffeedad8c5cec0392305e9bc775", - "sha256:c9bcad65d66d594bffab8575f39420fe0ee96f66e23c4d927ebb4e24354ec1af", - "sha256:ca9e8300d8ba0b66d140820cf463438c8e7b4cdc6fd710c059bfcfb1531d03fb", - "sha256:de4ecae89c7d8b56169473e08f6bfd2df7f95015591f43126e4ea7865928677e" + "sha256:0eb77764ea470f14fcbb89d51bc6bbf5e7623446ac4ed06cbd9ca9495b62e36e", + "sha256:1098df9a0592dd4c8c0ccfc2e98931278a6c6c53cb3a3e2cf7e9ee3b06153344", + "sha256:183b183b7771a508395d2cbffd6db67d6ad52958a5fdc99f450d954003900266", + "sha256:18fe320f354d6f9ad3147859b6e16649a0781425268c4dde596093177660e71a", + "sha256:26a432dc219c6b6f38be20a958cbe1abffcc5492821d7e27f08606ef99e0dffd", + "sha256:294a6903a4d087db805a7656989f613371915fc45c8cc0ddc5c5a0a8ad9bea4d", + "sha256:31d8c6b2df19a777bc8826770b872a45a1f30cfefcfd729491baa5237faae837", + "sha256:33b4a19ddc9fc551ebabca9765d54d04600c4a50eda13893dadf67ed81d9a098", + "sha256:42c47c3b43fe3a39ddf8de1d40dbbfca60ac8530a36c9b198ea5b9efac75c09e", + "sha256:525a2d4088e70a9f75b08b3f87a51acc9cde640e19cc523c7e41aa355564ae27", + "sha256:58ae097a325e9bb7a684572d20eb3e1809802c5c9ec7108e85da1eb6c1a3331b", + "sha256:676d051b1da67a852c0447621fdd11c4e104827417bf216092ec3e286f7da596", + "sha256:74cac86cc586db8dfda0ce65d8bcd2bf17b58668dfcc3652762f3ef0e6677e76", + "sha256:8c08d6625bb258179b6e512f55ad20f9dfef019bbfbe3095247401e053a3ea30", + "sha256:90904d889ab8e81a956f2c0935a523cc4e077c7847a836abee832f868d5c26a4", + "sha256:963a0ccc9a4188524e6e6d39b12c9ca24cc2d45a71cfdd04a26d883c922b4b78", + "sha256:bbebc31bf11762b63bf61aaae232becb41c5bf6b3461b80a4df7e791fabb3aca", + "sha256:bc2542e83ac8399752bc16e0b35e038bdb659ba237f4222616b4e83fb9654985", + "sha256:c29dd9a3a9d259c9fa19d19738d021632d673f6ed9b35a739f48e5f807f264fb", + "sha256:c7407cfcad702f0b6c0e0f3e7ab876cd1d2c13b14ce770e412c0c4b9728a0f88", + "sha256:da0a98d458010bf4fe535f2d1e367a2e2060e105978873c04c04212fb20543f7", + "sha256:df05aa5b241e2e8045f5f4367a9f6187b09c4cdf8578bb219861c4e27c443db5", + "sha256:f290617f74a610849bd8f5514e34ae3d09eafd521dceaa6cf68b3f4414266d4e", + "sha256:f30ddd110634c2d7534b2d4e0e22967e88366b0d356b24de87419cc4410c41b7" ], "markers": "python_version < '3.8'", - "version": "==1.5.1" + "version": "==1.5.2" }, "types-pyyaml": { "hashes": [ @@ -483,34 +508,34 @@ }, "types-requests": { "hashes": [ - "sha256:2e0e100dd489f83870d4f61949d3a7eae4821e7bfbf46c57e463c38f92d473d4", - "sha256:f38bd488528cdcbce5b01dc953972f3cead0d060cfd9ee35b363066c25bab13c" + "sha256:5d6f77f3c7565659bdb7b7bce1d33d1abb7d0b056138cac714860e13da2f19df", + "sha256:cf0646031dd6307113b37814f743c04f0707a3357378c2bb1326f848412f5ba9" ], "index": "pypi", - "version": "==2.27.7" + "version": "==2.27.13" }, "types-urllib3": { "hashes": [ - "sha256:3adcf2cb5981809091dbff456e6999fe55f201652d8c360f99997de5ac2f556e", - "sha256:cfd1fbbe4ba9a605ed148294008aac8a7b8b7472651d1cc357d507ae5962e3d2" + "sha256:24d64e441168851eb05f1d022de18ae31558f5649c8f1117e384c2e85e31315b", + "sha256:bd0abc01e9fb963e4fddd561a56d21cc371b988d1245662195c90379077139cd" ], - "version": "==1.26.7" + "version": "==1.26.11" }, "typing-extensions": { "hashes": [ - "sha256:4ca091dea149f945ec56afb48dae714f21e8692ef22a395223bcd328961b6a0e", - "sha256:7f001e5ac290a0c0401508864c7ec868be4e701886d5b573a9528ed3973d9d3b" + "sha256:1a9462dcc3347a79b1f1c0271fbe79e844580bb598bafa1ed208b94da3cdcd42", + "sha256:21c85e0fe4b9a155d0799430b0ad741cdce7e359660ccbd8b530613e8df88ce2" ], - "markers": "python_version < '3.8'", - "version": "==4.0.1" + "markers": "python_version >= '3.6'", + "version": "==4.1.1" }, "urllib3": { "hashes": [ - "sha256:000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed", - "sha256:0e7c33d9a63e7ddfcb86780aac87befc2fbddf46c58dbb487e0855f7ceec283c" + "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14", + "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", - "version": "==1.26.8" + "version": "==1.26.9" }, "validators": { "hashes": [ @@ -522,19 +547,19 @@ }, "virtualenv": { "hashes": [ - "sha256:339f16c4a86b44240ba7223d0f93a7887c3ca04b5f9c8129da7958447d079b09", - "sha256:d8458cf8d59d0ea495ad9b34c2599487f8a7772d796f9910858376d1600dd2dd" + "sha256:dd448d1ded9f14d1a4bfa6bfc0c5b96ae3be3f2d6c6c159b23ddcfd701baa021", + "sha256:e9dd1a1359d70137559034c0f5433b34caf504af2dc756367be86a5a32967134" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==20.13.0" + "version": "==20.13.3" }, "yamlfix": { "hashes": [ - "sha256:66094ed651d598baff809cfe9d0fdce447b70ad844c4319c18913a5ff58721af", - "sha256:b3d32734506bfbe28c4edea05f593e4a3a8ea4c1ad9b3490e971b0237ab6188d" + "sha256:ddcc14d6330c211d92a93d5662d3ef1cde61ab9b27d012f87f716a285425ccc8", + "sha256:ddf097c99fd572ccf6fb444227c0575308d16531cba7d231a689ccd6005e18a0" ], "index": "pypi", - "version": "==0.8.0" + "version": "==0.9.0" }, "yamllint": { "hashes": [ diff --git a/jenkins/opensearch/perf-test.jenkinsfile b/jenkins/opensearch/perf-test.jenkinsfile new file mode 100644 index 0000000000..bcd97905e6 --- /dev/null +++ b/jenkins/opensearch/perf-test.jenkinsfile @@ -0,0 +1,225 @@ +lib = library(identifier: "jenkins@20211118", retriever: legacySCM(scm)) + +pipeline { + agent none + options { + timeout(time: 10, unit: 'HOURS') + } + environment { + AGENT_LABEL = 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host' + AGENT_IMAGE = 'opensearchstaging/ci-runner:ci-runner-centos7-v1' + BUNDLE_MANIFEST = 'bundle-manifest.yml' + JOB_NAME = 'perf-test' + } + parameters { + string( + name: 'GITHUB_TOKEN', + description: 'Github token for account access.', + trim: true + ) + string( + name: 'BUNDLE_MANIFEST_URL', + description: 'The bundle manifest URL, e.g. https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.2.2/98/linux/x64/builds/opensearch/manifest.yml.', + trim: true + ) + string( + defaultValue: 'nyc_taxis', + name: 'TEST_WORKLOAD', + description: 'The workload name from OpenSearch Benchmark Workloads for Mensor (internal client).', + trim: true + ) + string( + defaultValue: '1', + name: 'TEST_ITERATIONS', + description: 'Number of times to run a workload for Mensor (internal client).', + trim: true + ) + string( + defaultValue: '0', + name: 'WARMUP_ITERATIONS', + description: 'Number of times to run a workload before collecting data for Mensor (internal client).', + trim: true + ) + } + + stages { + stage('validate-and-set-parameters') { + agent { + docker { + label AGENT_LABEL + image AGENT_IMAGE + alwaysPull true + } + } + steps { + script { + if (BUNDLE_MANIFEST_URL == '') { + currentBuild.result = 'ABORTED' + error("Performance Tests failed to start. Missing parameter: BUNDLE_MANIFEST_URL.") + } + if (GITHUB_TOKEN == '') { + currentBuild.result = 'ABORTED' + error("Performance Tests failed to start. Missing parameter: GITHUB_TOKEN.") + } + if (TEST_ITERATIONS != null && !TEST_ITERATIONS.isInteger()) { + currentBuild.result = 'ABORTED' + error("Performance Tests failed to start. Invalid value for parameter: TEST_ITERATIONS. Value should be an integer.") + } + if (WARMUP_ITERATIONS != null && !WARMUP_ITERATIONS.isInteger()) { + currentBuild.result = 'ABORTED' + error("Performance Tests failed to start. Invalid value for parameter: WARMUP_ITERATIONS. Value should be an integer.") + } + def bundleManifestObj = downloadBuildManifest( + url: BUNDLE_MANIFEST_URL, + path: BUNDLE_MANIFEST + ) + String buildId = bundleManifestObj.getArtifactBuildId() + env.BUILD_ID = buildId + env.HAS_SECURITY = bundleManifestObj.components.containsKey("security") + env.ARCHITECTURE = bundleManifestObj.getArtifactArchitecture() + echo "HAS_SECURITY: ${env.HAS_SECURITY}" + lib.jenkins.Messages.new(this).add(JOB_NAME, "Performance tests for #${BUILD_ID}") + } + } + } + stage('perf-test') { + parallel { + stage('test-with-security') { + agent { + docker { + label AGENT_LABEL + image AGENT_IMAGE + alwaysPull true + } + } + when { + expression { return env.HAS_SECURITY } + } + steps { + script { + def bundleManifestObj = downloadBuildManifest( + url: BUNDLE_MANIFEST_URL, + path: BUNDLE_MANIFEST + ) + echo "BUNDLE_MANIFEST: ${BUNDLE_MANIFEST}" + echo "BUILD_ID: ${BUILD_ID}" + echo "Architecture: ${ARCHITECTURE}" + + runPerfTestScript(bundleManifest: BUNDLE_MANIFEST, + buildId: BUILD_ID, + architecture: ARCHITECTURE, + insecure: false, + workload: TEST_WORKLOAD, + testIterations: TEST_ITERATIONS, + warmupIterations: WARMUP_ITERATIONS) + + lib.jenkins.Messages.new(this).add(JOB_NAME, + lib.jenkins.Messages.new(this).get([JOB_NAME]) + + "\nPerformance tests with security for ${BUILD_ID} completed") + } + } + post { + success { + script { + uploadTestResults( + buildManifestFileName: BUNDLE_MANIFEST, + jobName: JOB_NAME, + buildNumber: BUILD_ID + ) + } + postCleanup() + } + failure { + postCleanup() + } + aborted { + postCleanup() + } + } + } + stage('test-without-security') { + agent { + docker { + label AGENT_LABEL + image AGENT_IMAGE + alwaysPull true + } + } + steps { + script { + def bundleManifestObj = downloadBuildManifest( + url: BUNDLE_MANIFEST_URL, + path: BUNDLE_MANIFEST + ) + + echo "BUNDLE_MANIFEST: ${BUNDLE_MANIFEST}" + echo "BUILD_ID: ${BUILD_ID}" + echo "Architecture: ${ARCHITECTURE}" + + runPerfTestScript(bundleManifest: BUNDLE_MANIFEST, + buildId: BUILD_ID, + architecture: ARCHITECTURE, + insecure: true, + workload: TEST_WORKLOAD, + testIterations: TEST_ITERATIONS, + warmupIterations: WARMUP_ITERATIONS) + + lib.jenkins.Messages.new(this).add(JOB_NAME, + lib.jenkins.Messages.new(this).get([JOB_NAME]) + + "\nPerformance tests without security for ${BUILD_ID} completed") + } + } + post { + success { + script { + uploadTestResults( + buildManifestFileName: BUNDLE_MANIFEST, + jobName: JOB_NAME, + buildNumber: BUILD_ID + ) + } + postCleanup() + } + failure { + postCleanup() + } + aborted { + postCleanup() + } + } + } + } + } + } + + post { + success { + node(AGENT_LABEL) { + script { + def stashed = lib.jenkins.Messages.new(this).get([JOB_NAME]) + publishNotification( + icon: ':white_check_mark:', + message: 'Performance Tests Successful', + extra: stashed, + credentialsId: 'INTEG_TEST_WEBHOOK', + ) + postCleanup() + } + } + } + failure { + node(AGENT_LABEL) { + script { + def stashed = lib.jenkins.Messages.new(this).get([JOB_NAME]) + publishNotification( + icon: ':warning:', + message: 'Failed Performance Tests', + extra: stashed, + credentialsId: 'INTEG_TEST_WEBHOOK', + ) + postCleanup() + } + } + } + } +} \ No newline at end of file diff --git a/src/run_perf_test.py b/src/run_perf_test.py index 06aeb9a002..9be49ccce1 100644 --- a/src/run_perf_test.py +++ b/src/run_perf_test.py @@ -11,6 +11,7 @@ import sys import yaml +from retry.api import retry_call from git.git_repository import GitRepository from manifests.bundle_manifest import BundleManifest @@ -35,6 +36,9 @@ def main(): parser.add_argument("--bundle-manifest", type=argparse.FileType("r"), help="Bundle Manifest file.", required=True) parser.add_argument("--stack", dest="stack", help="Stack name for performance test") parser.add_argument("--config", type=argparse.FileType("r"), help="Config file.", required=True) + parser.add_argument("--without-security", dest="insecure", action="store_true", + help="Force the security of the cluster to be disabled.", + default=False) parser.add_argument("--keep", dest="keep", action="store_true", help="Do not delete the working temporary directory.") parser.add_argument("--workload", default="nyc_taxis", help="Mensor (internal client) param - Workload name from OpenSeach Benchmark Workloads") parser.add_argument("--workload-options", default="{}", help="Mensor (internal client) param - Json object with OpenSearch Benchmark arguments") @@ -45,14 +49,23 @@ def main(): manifest = BundleManifest.from_file(args.bundle_manifest) config = yaml.safe_load(args.config) + security = "security" in manifest.components and not args.insecure + tests_dir = os.path.join(os.getcwd(), "test-results", "perf-test", f"{'with' if security else 'without'}-security") + os.makedirs(tests_dir, exist_ok=True) + with TemporaryDirectory(keep=args.keep, chdir=True) as work_dir: current_workspace = os.path.join(work_dir.name, "infra") with GitRepository(get_infra_repo_url(), "main", current_workspace): - security = "security" in manifest.components with WorkingDirectory(current_workspace): - with PerfTestCluster.create(manifest, config, args.stack, security, current_workspace) as (test_cluster_endpoint, test_cluster_port): - perf_test_suite = PerfTestSuite(manifest, test_cluster_endpoint, security, current_workspace, args) - perf_test_suite.execute() + with PerfTestCluster.create( + manifest, + config, + args.stack, + security, + current_workspace) as (test_cluster_endpoint, test_cluster_port): + perf_test_suite = PerfTestSuite(manifest, test_cluster_endpoint, security, + current_workspace, tests_dir, args) + retry_call(perf_test_suite.execute, tries=3, delay=60, backoff=2) if __name__ == "__main__": diff --git a/src/test_workflow/perf_test/perf_test_suite.py b/src/test_workflow/perf_test/perf_test_suite.py index e379016f9d..d885690edd 100644 --- a/src/test_workflow/perf_test/perf_test_suite.py +++ b/src/test_workflow/perf_test/perf_test_suite.py @@ -8,8 +8,7 @@ class PerfTestSuite: """ Represents a performance test suite. This class runs rally test on the deployed cluster with the provided IP. """ - - def __init__(self, bundle_manifest, endpoint, security, current_workspace, args): + def __init__(self, bundle_manifest, endpoint, security, current_workspace, test_results_path, args): self.manifest = bundle_manifest self.work_dir = "mensor/" self.endpoint = endpoint @@ -18,21 +17,19 @@ def __init__(self, bundle_manifest, endpoint, security, current_workspace, args) self.args = args self.command = ( f"pipenv run python test_config.py -i {self.endpoint} -b {self.manifest.build.id}" - f" -a {self.manifest.build.architecture} -p {self.current_workspace}" + f" -a {self.manifest.build.architecture} -p {os.getcwd() if test_results_path is None else test_results_path}" f" --workload {self.args.workload} --workload-options '{self.args.workload_options}'" f" --warmup-iters {self.args.warmup_iters} --test-iters {self.args.test_iters}" ) def execute(self): try: - with WorkingDirectory(self.work_dir): - dir = os.getcwd() - subprocess.check_call("python3 -m pipenv install", cwd=dir, shell=True) - subprocess.check_call("pipenv install", cwd=dir, shell=True) - + current_workspace = os.path.join(self.current_workspace, self.work_dir) + with WorkingDirectory(current_workspace): + subprocess.check_call("pipenv install", cwd=current_workspace, shell=True) if self.security: - subprocess.check_call(f"{self.command} -s", cwd=dir, shell=True) + subprocess.check_call(f"{self.command} -s", cwd=current_workspace, shell=True) else: - subprocess.check_call(f"{self.command}", cwd=dir, shell=True) + subprocess.check_call(f"{self.command}", cwd=current_workspace, shell=True) finally: os.chdir(self.current_workspace) diff --git a/test.sh b/test.sh index 0c2c674dd2..c628e6a1e6 100755 --- a/test.sh +++ b/test.sh @@ -16,8 +16,11 @@ case $1 in "bwc-test") "$DIR/run.sh" "$DIR/src/run_bwc_test.py" "${@:2}" ;; + "perf-test") + "$DIR/run.sh" "$DIR/src/run_perf_test.py" "${@:2}" + ;; *) - echo "Invalid test suite, run ./test.sh integ-test|bwc-test." + echo "Invalid test suite, run ./test.sh integ-test|bwc-test|perf-test." exit 1 ;; esac diff --git a/tests/data/perf-test-config.yml b/tests/data/perf-test-config.yml new file mode 100644 index 0000000000..1625ca748e --- /dev/null +++ b/tests/data/perf-test-config.yml @@ -0,0 +1,9 @@ +--- +Description: Configuration file to store contants required to run rally for performance test +Constants: + Repository: https://github.com/opensearch-project/ + SecurityGroupId: test-security + VpcId: test-vpc + AccountId: 123456 + Region: eu-east-1 + Role: test-role-name diff --git a/tests/jenkins/TestRunNonSecurityPerfTestScript.groovy b/tests/jenkins/TestRunNonSecurityPerfTestScript.groovy new file mode 100644 index 0000000000..79f153d575 --- /dev/null +++ b/tests/jenkins/TestRunNonSecurityPerfTestScript.groovy @@ -0,0 +1,118 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +import jenkins.tests.BuildPipelineTest +import org.junit.Before +import org.junit.Test + +import static com.lesfurets.jenkins.unit.MethodCall.callArgsToString +import static org.hamcrest.CoreMatchers.equalTo +import static org.hamcrest.CoreMatchers.hasItem +import static org.hamcrest.MatcherAssert.assertThat + +class TestRunNonSecurityPerfTestScript extends BuildPipelineTest { + + @Before + void setUp() { + this.registerLibTester(new RunPerfTestScriptLibTester( + 'tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml', + '1236', + 'true', + 'nyc_taxis', + '1', + '1', + false + )) + super.setUp() + } + + @Test + public void testRunNonSecurityPerfTestScript_verifyPipeline() { + super.testPipeline("jenkins/opensearch/perf-test.jenkinsfile", + "tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile") + } + + @Test + void testRunNonSecurityPerfTestScript_verifyArtifactDownloads() { + runScript("jenkins/opensearch/perf-test.jenkinsfile") + + def curlCommands = getCommandExecutions('sh', 'curl').findAll { + shCommand -> shCommand.contains('curl') + } + + assertThat(curlCommands.size(), equalTo(3)) + assertThat(curlCommands, hasItem( + "curl test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml".toString() + )) + + def s3DownloadCommands = getCommandExecutions('s3Download', 'bucket').findAll { + shCommand -> shCommand.contains('bucket') + } + + assertThat(s3DownloadCommands.size(), equalTo(1)) + assertThat(s3DownloadCommands, hasItem( + "{file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true}".toString() + )) + } + + @Test + void testRunNonSecurityPerfTestScript_verifyPackageInstallation() { + runScript("jenkins/opensearch/perf-test.jenkinsfile") + + def npmCommands = getCommandExecutions('sh', 'npm').findAll { + shCommand -> shCommand.contains('npm') + } + + assertThat(npmCommands.size(), equalTo(1)) + + def pipenvCommands = getCommandExecutions('sh', 'pipenv').findAll { + shCommand -> shCommand.contains('pipenv') + } + + assertThat(pipenvCommands.size(), equalTo(1)) + + } + + @Test + void testRunNonSecurityPerfTestScript_verifyScriptExecutions() { + runScript("jenkins/opensearch/perf-test.jenkinsfile") + + def testScriptCommands = getCommandExecutions('sh', './test.sh').findAll { + shCommand -> shCommand.contains('./test.sh') + } + + assertThat(testScriptCommands.size(), equalTo(1)) + assertThat(testScriptCommands, hasItem( + "./test.sh perf-test --stack test-single-1236-x64 --bundle-manifest tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml --config config.yml --without-security --workload nyc_taxis --test-iters 1 --warmup-iters 1".toString() + )) + + def resultUploadScriptCommands = getCommandExecutions('s3Upload', 'test-results').findAll { + shCommand -> shCommand.contains('test-results') + } + assertThat(resultUploadScriptCommands.size(), equalTo(1)) + assertThat(resultUploadScriptCommands, hasItem( + "{file=test-results, bucket=test_bucket, path=perf-test/1.3.0/1236/linux/x64/test-results}".toString() + )) + } + + def getCommandExecutions(methodName, command) { + def shCommands = helper.callStack.findAll { + call -> + call.methodName == methodName + }. + collect { + call -> + callArgsToString(call) + }.findAll { + shCommand -> + shCommand.contains(command) + } + + return shCommands + } +} \ No newline at end of file diff --git a/tests/jenkins/TestRunPerfTestScript.groovy b/tests/jenkins/TestRunPerfTestScript.groovy new file mode 100644 index 0000000000..5520ef1940 --- /dev/null +++ b/tests/jenkins/TestRunPerfTestScript.groovy @@ -0,0 +1,121 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +import jenkins.tests.BuildPipelineTest +import org.junit.Before +import org.junit.Test + +import static com.lesfurets.jenkins.unit.MethodCall.callArgsToString +import static org.hamcrest.CoreMatchers.equalTo +import static org.hamcrest.CoreMatchers.hasItem +import static org.hamcrest.MatcherAssert.assertThat + +class TestRunPerfTestScript extends BuildPipelineTest { + + @Before + void setUp() { + this.registerLibTester(new RunPerfTestScriptLibTester( + 'tests/jenkins/data/opensearch-1.3.0-bundle.yml', + '1236', + 'true', + 'nyc_taxis', + '1', + '1', + true + )) + super.setUp() + } + + @Test + public void testRunPerfTestScript_Pipeline() { + super.testPipeline("jenkins/opensearch/perf-test.jenkinsfile", + "tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile") + } + + @Test + void testRunPerfTestScript_verifyArtifactDownloads() { + runScript("jenkins/opensearch/perf-test.jenkinsfile") + + def curlCommands = getCommandExecutions('sh', 'curl').findAll { + shCommand -> shCommand.contains('curl') + } + + assertThat(curlCommands.size(), equalTo(4)) + assertThat(curlCommands, hasItem( + "curl test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-bundle.yml".toString() + )) + + def s3DownloadCommands = getCommandExecutions('s3Download', 'bucket').findAll { + shCommand -> shCommand.contains('bucket') + } + + assertThat(s3DownloadCommands.size(), equalTo(2)) + assertThat(s3DownloadCommands, hasItem( + "{file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true}".toString() + )) + } + + @Test + void testRunPerfTestScript_verifyPackageInstallation() { + runScript("jenkins/opensearch/perf-test.jenkinsfile") + + def npmCommands = getCommandExecutions('sh', 'npm').findAll { + shCommand -> shCommand.contains('npm') + } + + assertThat(npmCommands.size(), equalTo(2)) + + def pipenvCommands = getCommandExecutions('sh', 'pipenv').findAll { + shCommand -> shCommand.contains('pipenv') + } + + assertThat(pipenvCommands.size(), equalTo(2)) + + } + + @Test + void testRunPerfTestScript_verifyScriptExecutions() { + runScript("jenkins/opensearch/perf-test.jenkinsfile") + + def testScriptCommands = getCommandExecutions('sh', './test.sh').findAll { + shCommand -> shCommand.contains('./test.sh') + } + + assertThat(testScriptCommands.size(), equalTo(2)) + assertThat(testScriptCommands, hasItem( + "./test.sh perf-test --stack test-single-security-1236-x64 --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config config.yml --workload nyc_taxis --test-iters 1 --warmup-iters 1".toString() + )) + assertThat(testScriptCommands, hasItem( + "./test.sh perf-test --stack test-single-1236-x64 --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config config.yml --without-security --workload nyc_taxis --test-iters 1 --warmup-iters 1".toString() + )) + + def resultUploadScriptCommands = getCommandExecutions('s3Upload', 'test-results').findAll { + shCommand -> shCommand.contains('test-results') + } + assertThat(resultUploadScriptCommands.size(), equalTo(2)) + assertThat(resultUploadScriptCommands, hasItem( + "{file=test-results, bucket=test_bucket, path=perf-test/1.3.0/1236/linux/x64/test-results}".toString() + )) + } + + def getCommandExecutions(methodName, command) { + def shCommands = helper.callStack.findAll { + call -> + call.methodName == methodName + }. + collect { + call -> + callArgsToString(call) + }.findAll { + shCommand -> + shCommand.contains(command) + } + + return shCommands + } +} \ No newline at end of file diff --git a/tests/jenkins/data/opensearch-1.3.0-bundle.yml b/tests/jenkins/data/opensearch-1.3.0-bundle.yml new file mode 100644 index 0000000000..d5ee76eaee --- /dev/null +++ b/tests/jenkins/data/opensearch-1.3.0-bundle.yml @@ -0,0 +1,21 @@ +--- +schema-version: '1.1' +build: + name: OpenSearch + version: 1.3.0 + platform: linux + architecture: x64 + distribution: tar + location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/1236/linux/x64/dist/opensearch/opensearch-1.3.0-linux-x64.tar.gz + id: '1236' +components: + - name: OpenSearch + repository: https://github.com/opensearch-project/OpenSearch.git + ref: 1.x + commit_id: 22408088f002a4fc8cdd3b2ed7438866c14c5069 + location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/1236/linux/x64/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64.tar.gz + - name: security + repository: https://github.com/opensearch-project/security.git + ref: 1.x + commit_id: 9bf0ab861eae018352427e44b434aa12c1577fa5 + location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/1236/linux/x64/builds/opensearch/plugins/opensearch-security-1.3.0.0.zip diff --git a/tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml b/tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml new file mode 100644 index 0000000000..8a70c63e74 --- /dev/null +++ b/tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml @@ -0,0 +1,16 @@ +--- +schema-version: '1.1' +build: + name: OpenSearch + version: 1.3.0 + platform: linux + architecture: x64 + distribution: tar + location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/1236/linux/x64/dist/opensearch/opensearch-1.3.0-linux-x64.tar.gz + id: '1236' +components: + - name: OpenSearch + repository: https://github.com/opensearch-project/OpenSearch.git + ref: 1.x + commit_id: 22408088f002a4fc8cdd3b2ed7438866c14c5069 + location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.3.0/1236/linux/x64/builds/opensearch/dist/opensearch-min-1.3.0-linux-x64.tar.gz diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile.txt new file mode 100644 index 0000000000..cfed0d2dd0 --- /dev/null +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile.txt @@ -0,0 +1,164 @@ + perf-test.run() + perf-test.legacySCM(groovy.lang.Closure) + perf-test.library({identifier=jenkins@20211118, retriever=null}) + perf-test.pipeline(groovy.lang.Closure) + perf-test.timeout({time=10, unit=HOURS}) + perf-test.echo(Executing on agent [label:none]) + perf-test.stage(validate-and-set-parameters, groovy.lang.Closure) + perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) + perf-test.script(groovy.lang.Closure) + perf-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) + downloadBuildManifest.legacySCM(groovy.lang.Closure) + downloadBuildManifest.library({identifier=jenkins@20211123, retriever=null}) + downloadBuildManifest.sh(curl test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-bundle.yml) + downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) + BuildManifest.asBoolean() + BuildManifest.getArtifactBuildId() + BuildManifest.getArtifactArchitecture() + perf-test.echo(HAS_SECURITY: true) + Messages.asBoolean() + Messages.add(perf-test, Performance tests for #1236) + perf-test.writeFile({file=messages/perf-test.msg, text=Performance tests for #1236}) + perf-test.stash({includes=messages/*, name=messages-perf-test}) + perf-test.stage(test-with-security, groovy.lang.Closure) + perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) + perf-test.script(groovy.lang.Closure) + perf-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) + downloadBuildManifest.legacySCM(groovy.lang.Closure) + downloadBuildManifest.library({identifier=jenkins@20211123, retriever=null}) + downloadBuildManifest.sh(curl test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-bundle.yml) + downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) + BuildManifest.asBoolean() + perf-test.echo(BUNDLE_MANIFEST: tests/jenkins/data/opensearch-1.3.0-bundle.yml) + perf-test.echo(BUILD_ID: 1236) + perf-test.echo(Architecture: x64) + perf-test.runPerfTestScript({bundleManifest=tests/jenkins/data/opensearch-1.3.0-bundle.yml, buildId=1236, architecture=x64, insecure=false, workload=nyc_taxis, testIterations=1, warmupIterations=1}) + runPerfTestScript.legacySCM(groovy.lang.Closure) + runPerfTestScript.library({identifier=jenkins@20211123, retriever=null}) + runPerfTestScript.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) + BuildManifest.asBoolean() + runPerfTestScript.sh( + npm install -g fs-extra + npm install -g chalk@4.1.2 + npm install -g @aws-cdk/cloudformation-diff + npm install -g aws-cdk + npm install -g cdk-assume-role-credential-plugin@1.4.0 + ) + runPerfTestScript.sh( + pipenv install "dataclasses_json~=0.5" "aws_requests_auth~=0.4" "json2html~=1.3.0" + pipenv install "aws-cdk.core~=1.143.0" "aws_cdk.aws_ec2~=1.143.0" "aws_cdk.aws_iam~=1.143.0" + pipenv install "boto3~=1.18" "setuptools~=57.4" "retry~=0.9" + ) + runPerfTestScript.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + runPerfTestScript.s3Download({file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true}) + runPerfTestScript.sh(./test.sh perf-test --stack test-single-security-1236-x64 --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config config.yml --workload nyc_taxis --test-iters 1 --warmup-iters 1) + Messages.asBoolean() + Messages.asBoolean() + Messages.get([perf-test]) + perf-test.unstash({name=messages-perf-test}) + perf-test.findFiles({excludes=, glob=messages/*}) + perf-test.dir(messages, groovy.lang.Closure) + perf-test.deleteDir() + Messages.add(perf-test, +Performance tests with security for 1236 completed) + perf-test.writeFile({file=messages/perf-test.msg, text= +Performance tests with security for 1236 completed}) + perf-test.stash({includes=messages/*, name=messages-perf-test}) + perf-test.script(groovy.lang.Closure) + perf-test.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-1.3.0-bundle.yml, jobName=perf-test, buildNumber=1236}) + uploadTestResults.legacySCM(groovy.lang.Closure) + uploadTestResults.library({identifier=jenkins@20211123, retriever=null}) + uploadTestResults.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) + BuildManifest.asBoolean() + BuildManifest.getArtifactRoot(perf-test, 1236) + uploadTestResults.echo(Uploading to s3://test_bucket/perf-test/1.3.0/1236/linux/x64) + uploadTestResults.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + uploadTestResults.s3Upload({file=test-results, bucket=test_bucket, path=perf-test/1.3.0/1236/linux/x64/test-results}) + BuildManifest.getArtifactRootUrl(test://artifact.url, perf-test, 1236) + Messages.asBoolean() + Messages.add(test_stage, test://artifact.url/perf-test/1.3.0/1236/linux/x64/test-results/) + uploadTestResults.writeFile({file=messages/test_stage.msg, text=test://artifact.url/perf-test/1.3.0/1236/linux/x64/test-results/}) + uploadTestResults.stash({includes=messages/*, name=messages-test_stage}) + perf-test.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) + perf-test.stage(test-without-security, groovy.lang.Closure) + perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) + perf-test.script(groovy.lang.Closure) + perf-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) + downloadBuildManifest.legacySCM(groovy.lang.Closure) + downloadBuildManifest.library({identifier=jenkins@20211123, retriever=null}) + downloadBuildManifest.sh(curl test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-bundle.yml) + downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) + BuildManifest.asBoolean() + perf-test.echo(BUNDLE_MANIFEST: tests/jenkins/data/opensearch-1.3.0-bundle.yml) + perf-test.echo(BUILD_ID: 1236) + perf-test.echo(Architecture: x64) + perf-test.runPerfTestScript({bundleManifest=tests/jenkins/data/opensearch-1.3.0-bundle.yml, buildId=1236, architecture=x64, insecure=true, workload=nyc_taxis, testIterations=1, warmupIterations=1}) + runPerfTestScript.legacySCM(groovy.lang.Closure) + runPerfTestScript.library({identifier=jenkins@20211123, retriever=null}) + runPerfTestScript.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) + BuildManifest.asBoolean() + runPerfTestScript.sh( + npm install -g fs-extra + npm install -g chalk@4.1.2 + npm install -g @aws-cdk/cloudformation-diff + npm install -g aws-cdk + npm install -g cdk-assume-role-credential-plugin@1.4.0 + ) + runPerfTestScript.sh( + pipenv install "dataclasses_json~=0.5" "aws_requests_auth~=0.4" "json2html~=1.3.0" + pipenv install "aws-cdk.core~=1.143.0" "aws_cdk.aws_ec2~=1.143.0" "aws_cdk.aws_iam~=1.143.0" + pipenv install "boto3~=1.18" "setuptools~=57.4" "retry~=0.9" + ) + runPerfTestScript.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + runPerfTestScript.s3Download({file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true}) + runPerfTestScript.sh(./test.sh perf-test --stack test-single-1236-x64 --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config config.yml --without-security --workload nyc_taxis --test-iters 1 --warmup-iters 1) + Messages.asBoolean() + Messages.asBoolean() + Messages.get([perf-test]) + perf-test.unstash({name=messages-perf-test}) + perf-test.findFiles({excludes=, glob=messages/*}) + perf-test.dir(messages, groovy.lang.Closure) + perf-test.deleteDir() + Messages.add(perf-test, +Performance tests without security for 1236 completed) + perf-test.writeFile({file=messages/perf-test.msg, text= +Performance tests without security for 1236 completed}) + perf-test.stash({includes=messages/*, name=messages-perf-test}) + perf-test.script(groovy.lang.Closure) + perf-test.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-1.3.0-bundle.yml, jobName=perf-test, buildNumber=1236}) + uploadTestResults.legacySCM(groovy.lang.Closure) + uploadTestResults.library({identifier=jenkins@20211123, retriever=null}) + uploadTestResults.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) + BuildManifest.asBoolean() + BuildManifest.getArtifactRoot(perf-test, 1236) + uploadTestResults.echo(Uploading to s3://test_bucket/perf-test/1.3.0/1236/linux/x64) + uploadTestResults.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + uploadTestResults.s3Upload({file=test-results, bucket=test_bucket, path=perf-test/1.3.0/1236/linux/x64/test-results}) + BuildManifest.getArtifactRootUrl(test://artifact.url, perf-test, 1236) + Messages.asBoolean() + Messages.add(test_stage, test://artifact.url/perf-test/1.3.0/1236/linux/x64/test-results/) + uploadTestResults.writeFile({file=messages/test_stage.msg, text=test://artifact.url/perf-test/1.3.0/1236/linux/x64/test-results/}) + uploadTestResults.stash({includes=messages/*, name=messages-test_stage}) + perf-test.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) + perf-test.node(Jenkins-Agent-al2-x64-c54xlarge-Docker-Host, groovy.lang.Closure) + perf-test.script(groovy.lang.Closure) + Messages.asBoolean() + Messages.get([perf-test]) + perf-test.unstash({name=messages-perf-test}) + perf-test.findFiles({excludes=, glob=messages/*}) + perf-test.dir(messages, groovy.lang.Closure) + perf-test.deleteDir() + perf-test.publishNotification({icon=:white_check_mark:, message=Performance Tests Successful, extra=, credentialsId=INTEG_TEST_WEBHOOK}) + publishNotification.string({credentialsId=INTEG_TEST_WEBHOOK, variable=WEBHOOK_URL}) + publishNotification.withCredentials([WEBHOOK_URL], groovy.lang.Closure) + publishNotification.sh(curl -XPOST --header "Content-Type: application/json" --data '{"result_text":":white_check_mark: +JOB_NAME=perf-test +BUILD_NUMBER=[1236] +MESSAGE=Performance Tests Successful +BUILD_URL: test://artifact.url +MANIFEST: null +"}' "WEBHOOK_URL") + perf-test.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt new file mode 100644 index 0000000000..e426dca7cd --- /dev/null +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile.txt @@ -0,0 +1,104 @@ + perf-test.run() + perf-test.legacySCM(groovy.lang.Closure) + perf-test.library({identifier=jenkins@20211118, retriever=null}) + perf-test.pipeline(groovy.lang.Closure) + perf-test.timeout({time=10, unit=HOURS}) + perf-test.echo(Executing on agent [label:none]) + perf-test.stage(validate-and-set-parameters, groovy.lang.Closure) + perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) + perf-test.script(groovy.lang.Closure) + perf-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) + downloadBuildManifest.legacySCM(groovy.lang.Closure) + downloadBuildManifest.library({identifier=jenkins@20211123, retriever=null}) + downloadBuildManifest.sh(curl test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml) + downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) + BuildManifest.asBoolean() + BuildManifest.getArtifactBuildId() + BuildManifest.getArtifactArchitecture() + perf-test.echo(HAS_SECURITY: false) + Messages.asBoolean() + Messages.add(perf-test, Performance tests for #1236) + perf-test.writeFile({file=messages/perf-test.msg, text=Performance tests for #1236}) + perf-test.stash({includes=messages/*, name=messages-perf-test}) + perf-test.echo(Skipping stage test-with-security) + perf-test.stage(test-without-security, groovy.lang.Closure) + perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]]) + perf-test.script(groovy.lang.Closure) + perf-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) + downloadBuildManifest.legacySCM(groovy.lang.Closure) + downloadBuildManifest.library({identifier=jenkins@20211123, retriever=null}) + downloadBuildManifest.sh(curl test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml) + downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) + BuildManifest.asBoolean() + perf-test.echo(BUNDLE_MANIFEST: tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml) + perf-test.echo(BUILD_ID: 1236) + perf-test.echo(Architecture: x64) + perf-test.runPerfTestScript({bundleManifest=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml, buildId=1236, architecture=x64, insecure=true, workload=nyc_taxis, testIterations=1, warmupIterations=1}) + runPerfTestScript.legacySCM(groovy.lang.Closure) + runPerfTestScript.library({identifier=jenkins@20211123, retriever=null}) + runPerfTestScript.readYaml({file=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) + BuildManifest.asBoolean() + runPerfTestScript.sh( + npm install -g fs-extra + npm install -g chalk@4.1.2 + npm install -g @aws-cdk/cloudformation-diff + npm install -g aws-cdk + npm install -g cdk-assume-role-credential-plugin@1.4.0 + ) + runPerfTestScript.sh( + pipenv install "dataclasses_json~=0.5" "aws_requests_auth~=0.4" "json2html~=1.3.0" + pipenv install "aws-cdk.core~=1.143.0" "aws_cdk.aws_ec2~=1.143.0" "aws_cdk.aws_iam~=1.143.0" + pipenv install "boto3~=1.18" "setuptools~=57.4" "retry~=0.9" + ) + runPerfTestScript.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + runPerfTestScript.s3Download({file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true}) + runPerfTestScript.sh(./test.sh perf-test --stack test-single-1236-x64 --bundle-manifest tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml --config config.yml --without-security --workload nyc_taxis --test-iters 1 --warmup-iters 1) + Messages.asBoolean() + Messages.asBoolean() + Messages.get([perf-test]) + perf-test.unstash({name=messages-perf-test}) + perf-test.findFiles({excludes=, glob=messages/*}) + perf-test.dir(messages, groovy.lang.Closure) + perf-test.deleteDir() + Messages.add(perf-test, +Performance tests without security for 1236 completed) + perf-test.writeFile({file=messages/perf-test.msg, text= +Performance tests without security for 1236 completed}) + perf-test.stash({includes=messages/*, name=messages-perf-test}) + perf-test.script(groovy.lang.Closure) + perf-test.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml, jobName=perf-test, buildNumber=1236}) + uploadTestResults.legacySCM(groovy.lang.Closure) + uploadTestResults.library({identifier=jenkins@20211123, retriever=null}) + uploadTestResults.readYaml({file=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) + BuildManifest.asBoolean() + BuildManifest.getArtifactRoot(perf-test, 1236) + uploadTestResults.echo(Uploading to s3://test_bucket/perf-test/1.3.0/1236/linux/x64) + uploadTestResults.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + uploadTestResults.s3Upload({file=test-results, bucket=test_bucket, path=perf-test/1.3.0/1236/linux/x64/test-results}) + BuildManifest.getArtifactRootUrl(test://artifact.url, perf-test, 1236) + Messages.asBoolean() + Messages.add(test_stage, test://artifact.url/perf-test/1.3.0/1236/linux/x64/test-results/) + uploadTestResults.writeFile({file=messages/test_stage.msg, text=test://artifact.url/perf-test/1.3.0/1236/linux/x64/test-results/}) + uploadTestResults.stash({includes=messages/*, name=messages-test_stage}) + perf-test.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) + perf-test.node(Jenkins-Agent-al2-x64-c54xlarge-Docker-Host, groovy.lang.Closure) + perf-test.script(groovy.lang.Closure) + Messages.asBoolean() + Messages.get([perf-test]) + perf-test.unstash({name=messages-perf-test}) + perf-test.findFiles({excludes=, glob=messages/*}) + perf-test.dir(messages, groovy.lang.Closure) + perf-test.deleteDir() + perf-test.publishNotification({icon=:white_check_mark:, message=Performance Tests Successful, extra=, credentialsId=INTEG_TEST_WEBHOOK}) + publishNotification.string({credentialsId=INTEG_TEST_WEBHOOK, variable=WEBHOOK_URL}) + publishNotification.withCredentials([WEBHOOK_URL], groovy.lang.Closure) + publishNotification.sh(curl -XPOST --header "Content-Type: application/json" --data '{"result_text":":white_check_mark: +JOB_NAME=perf-test +BUILD_NUMBER=[1236] +MESSAGE=Performance Tests Successful +BUILD_URL: test://artifact.url +MANIFEST: null +"}' "WEBHOOK_URL") + perf-test.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) diff --git a/tests/jenkins/lib-testers/RunPerfTestScriptLibTest.groovy b/tests/jenkins/lib-testers/RunPerfTestScriptLibTest.groovy new file mode 100644 index 0000000000..242eb34f9c --- /dev/null +++ b/tests/jenkins/lib-testers/RunPerfTestScriptLibTest.groovy @@ -0,0 +1,85 @@ +import static org.hamcrest.CoreMatchers.notNullValue +import static org.hamcrest.MatcherAssert.assertThat + +class RunPerfTestScriptLibTester extends LibFunctionTester { + + private String bundleManifest + private String buildId + private String insecure + private String workload + private String testIterations + private String warmupIterations + private boolean security + + public RunPerfTestScriptLibTester(bundleManifest, buildId, insecure, workload, + testIterations, warmupIterations, security) { + this.bundleManifest = bundleManifest + this.buildId = buildId + this.insecure = insecure + this.workload = workload + this.testIterations = testIterations + this.warmupIterations = warmupIterations + this.security = security + } + + void configure(helper, binding) { + helper.registerAllowedMethod("s3Download", [Map]) + helper.registerAllowedMethod("uploadTestResults", [Map]) + helper.registerAllowedMethod("s3Upload", [Map]) + helper.registerAllowedMethod("withAWS", [Map, Closure], { + args, + closure -> + closure.delegate = delegate + return helper.callClosure(closure) + }) + helper.registerAllowedMethod('findFiles', [Map.class], null) + helper.registerAllowedMethod("withCredentials", [Map]) + helper.registerAllowedMethod("downloadBuildManifest", [Map], { + c -> lib.jenkins.BuildManifest.new(readYaml(file: bundleManifest)) + }) + + binding.setVariable('AGENT_LABEL', 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host') + binding.setVariable('AGENT_IMAGE', 'opensearchstaging/ci-runner:ci-runner-centos7-v1') + binding.setVariable('ARCHITECTURE', 'x64') + binding.setVariable('ARTIFACT_BUCKET_NAME', 'test_bucket') + binding.setVariable('ARTIFACT_DOWNLOAD_ROLE_NAME', 'Dummy_Download_Role') + binding.setVariable('AWS_ACCOUNT_PUBLIC', 'dummy_account') + binding.setVariable('BUILD_ID', buildId) + binding.setVariable('BUILD_NUMBER', buildId) + binding.setVariable('BUILD_URL', 'test://artifact.url') + binding.setVariable('BUNDLE_MANIFEST', bundleManifest) + binding.setVariable('BUNDLE_MANIFEST_URL', 'test://artifact.url') + binding.setVariable('GITHUB_TOKEN', 'test_token') + binding.setVariable('HAS_SECURITY', security) + binding.setVariable('JOB_NAME', 'perf-test') + binding.setVariable('PERF_TEST_CONFIG_LOCATION', 'test_config') + binding.setVariable('PUBLIC_ARTIFACT_URL', 'test://artifact.url') + binding.setVariable('STAGE_NAME', 'test_stage') + binding.setVariable('TEST_ITERATIONS', testIterations) + binding.setVariable('TEST_WORKLOAD', workload) + binding.setVariable('WEBHOOK_URL', 'test://artifact.url') + binding.setVariable('WARMUP_ITERATIONS', warmupIterations) + + } + + void parameterInvariantsAssertions(call) { + assertThat(call.args.bundleManifest.first(), notNullValue()) + assertThat(call.args.buildId.first(), notNullValue()) + assertThat(call.args.insecure.first(), notNullValue()) + assertThat(call.args.workload.first(), notNullValue()) + assertThat(call.args.testIterations.first(), notNullValue()) + assertThat(call.args.warmupIterations.first(), notNullValue()) + } + + boolean expectedParametersMatcher(call) { + return call.args.bundleManifest.first().toString().equals(this.bundleManifest) && + call.args.buildId.first().toString().equals(this.buildId) && + call.args.workload.first().toString().equals(this.workload) && + call.args.testIterations.first().toInteger().equals(this.testIterations.toInteger()) && + call.args.warmupIterations.first().toInteger().equals(this.warmupIterations.toInteger()) + } + + String libFunctionName() { + return 'runPerfTestScript' + } +} \ No newline at end of file diff --git a/tests/test_run_perf_test.py b/tests/test_run_perf_test.py new file mode 100644 index 0000000000..d96f0063d9 --- /dev/null +++ b/tests/test_run_perf_test.py @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. + +import os +import tempfile +import unittest +from unittest.mock import Mock, patch + +import pytest + +from run_perf_test import main + + +class TestRunPerfTest(unittest.TestCase): + @pytest.fixture(autouse=True) + def capfd(self, capfd): + self.capfd = capfd + + @patch("argparse._sys.argv", ["run_perf_test.py", "--help"]) + def test_usage(self): + with self.assertRaises(SystemExit): + main() + + out, _ = self.capfd.readouterr() + self.assertTrue(out.startswith("usage:")) + + BUNDLE_MANIFEST_PATH = os.path.join( + os.path.dirname(__file__), + "jenkins", + "data", + ) + + CONFIG_ROOT_PATH = os.path.join( + os.path.dirname(__file__), + "data", + ) + + OPENSEARCH_BUNDLE_MANIFEST = os.path.realpath(os.path.join(BUNDLE_MANIFEST_PATH, "opensearch-1.3.0-bundle.yml")) + PERF_TEST_CONFIG = os.path.realpath(os.path.join(CONFIG_ROOT_PATH, "perf-test-config.yml")) + + @patch("argparse._sys.argv", ["run_perf_test.py", "--bundle-manifest", OPENSEARCH_BUNDLE_MANIFEST, + "--stack", "test-stack", "--config", PERF_TEST_CONFIG]) + @patch("run_perf_test.PerfTestCluster.create") + @patch("run_perf_test.PerfTestSuite") + @patch("run_perf_test.WorkingDirectory") + @patch("run_perf_test.TemporaryDirectory") + @patch("run_perf_test.GitRepository") + def test_default_execute_perf_test(self, mock_git, mock_temp, mock_working_dir, mock_suite, mock_cluster, *mocks): + mock_temp.return_value.__enter__.return_value.name = tempfile.gettempdir() + mock_create = Mock() + mock_create.__enter__ = Mock(return_value=('test-endpoint', 1234)) + mock_create.__exit__ = Mock(return_value=None) + mock_cluster.return_value = mock_create + + mock_execute = Mock() + mock_suite.return_value.execute = mock_execute + + main() + self.assertEqual(1, mock_cluster.call_count) + self.assertEqual(1, mock_suite.call_count) + self.assertEqual(1, mock_git.call_count) + self.assertEqual(1, mock_execute.call_count) + self.assertEqual([], mock_execute.call_args) + self.assertIn(True, mock_suite.call_args[0]) + + @patch("argparse._sys.argv", ["run_perf_test.py", "--bundle-manifest", OPENSEARCH_BUNDLE_MANIFEST, + "--stack", "test-stack", "--config", PERF_TEST_CONFIG, "--without-security"]) + @patch("run_perf_test.PerfTestCluster.create") + @patch("run_perf_test.PerfTestSuite") + @patch("run_perf_test.WorkingDirectory") + @patch("run_perf_test.TemporaryDirectory") + @patch("run_perf_test.GitRepository") + def test_with_security_execute_perf_test(self, mock_git, mock_temp, mock_working_dir, mock_suite, mock_cluster, *mocks): + mock_temp.return_value.__enter__.return_value.name = tempfile.gettempdir() + mock_create = Mock() + mock_create.__enter__ = Mock(return_value=('test-endpoint', 1234)) + mock_create.__exit__ = Mock(return_value=None) + mock_cluster.return_value = mock_create + + mock_execute = Mock() + mock_suite.return_value.execute = mock_execute + + main() + self.assertEqual(1, mock_cluster.call_count) + self.assertEqual(1, mock_suite.call_count) + self.assertEqual(1, mock_git.call_count) + self.assertEqual(1, mock_execute.call_count) + self.assertEqual([], mock_execute.call_args) + self.assertIn(False, mock_suite.call_args[0]) diff --git a/tests/tests_test_workflow/test_perf_workflow/perf_test/test_perf_test_suite.py b/tests/tests_test_workflow/test_perf_workflow/perf_test/test_perf_test_suite.py index 0d08947a12..35e244485f 100644 --- a/tests/tests_test_workflow/test_perf_workflow/perf_test/test_perf_test_suite.py +++ b/tests/tests_test_workflow/test_perf_workflow/perf_test/test_perf_test_suite.py @@ -23,10 +23,13 @@ def setUp(self): self.manifest_filename = os.path.join(self.data_path, "bundle_manifest.yml") self.manifest = BundleManifest.from_path(self.manifest_filename) self.endpoint = None - self.perf_test_suite = PerfTestSuite(bundle_manifest=self.manifest, endpoint=None, security=False, current_workspace="current_workspace", args=self.args) + + self.perf_test_suite = PerfTestSuite(bundle_manifest=self.manifest, endpoint=None, security=False, + current_workspace="current_workspace", test_results_path="test/results/", + args=self.args) def test_execute(self): with patch("test_workflow.perf_test.perf_test_suite.os.chdir"): with patch("subprocess.check_call") as mock_check_call: self.perf_test_suite.execute() - self.assertEqual(mock_check_call.call_count, 3) + self.assertEqual(mock_check_call.call_count, 2) diff --git a/vars/runPerfTestScript.groovy b/vars/runPerfTestScript.groovy new file mode 100644 index 0000000000..bec1267c2c --- /dev/null +++ b/vars/runPerfTestScript.groovy @@ -0,0 +1,43 @@ +void call(Map args = [:]) { + lib = library(identifier: 'jenkins@20211123', retriever: legacySCM(scm)) + def buildManifest = lib.jenkins.BuildManifest.new(readYaml(file: args.bundleManifest)) + + install_dependencies() + install_opensearch_infra_dependencies() + withAWS(role: 'opensearch-test', roleAccount: "${AWS_ACCOUNT_PUBLIC}", duration: 900, roleSessionName: 'jenkins-session') { + s3Download(file: "config.yml", bucket: "${ARTIFACT_BUCKET_NAME}", path: "${PERF_TEST_CONFIG_LOCATION}/config.yml", force: true) + } + + sh([ + './test.sh', + 'perf-test', + args.insecure ? "--stack test-single-${args.buildId}-${args.architecture}" : + "--stack test-single-security-${args.buildId}-${args.architecture}", + "--bundle-manifest ${args.bundleManifest}", + "--config config.yml", + args.insecure ? "--without-security" : "", + isNullOrEmpty(args.workload) ? "" : "--workload ${args.workload}", + isNullOrEmpty(args.testIterations) ? "" : "--test-iters ${args.testIterations}", + isNullOrEmpty(args.warmupIterations) ? "" : "--warmup-iters ${args.warmupIterations}", + ].join(' ')) +} + +boolean isNullOrEmpty(String str) { return (str == null || str.allWhitespace) } + +void install_opensearch_infra_dependencies() { + sh''' + pipenv install "dataclasses_json~=0.5" "aws_requests_auth~=0.4" "json2html~=1.3.0" + pipenv install "aws-cdk.core~=1.143.0" "aws_cdk.aws_ec2~=1.143.0" "aws_cdk.aws_iam~=1.143.0" + pipenv install "boto3~=1.18" "setuptools~=57.4" "retry~=0.9" + ''' +} + +void install_dependencies() { + sh ''' + npm install -g fs-extra + npm install -g chalk@4.1.2 + npm install -g @aws-cdk/cloudformation-diff + npm install -g aws-cdk + npm install -g cdk-assume-role-credential-plugin@1.4.0 + ''' +} \ No newline at end of file