Skip to content

Commit

Permalink
Run tests against actual Jenkins perf-test job
Browse files Browse the repository at this point in the history 
Signed-off-by: Kunal Kotwani <[email protected]>
  • Loading branch information
@kotwanikunal
kotwanikunal committed Mar 25, 2022
1 parent 75e8d53 commit 3f98feb
Show file tree
Hide file tree
Showing 9 changed files with 198 additions and 475 deletions.
5 changes: 1 addition & 4 deletions jenkins/opensearch/perf-test.jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ pipeline {
}
}
when {
environment name: 'HAS_SECURITY', value: 'true'
expression { HAS_SECURITY == true }
}
steps {
script {
Expand All @@ -100,8 +100,6 @@ pipeline {
)
echo "BUNDLE_MANIFEST: ${BUNDLE_MANIFEST}"
echo "BUILD_ID: ${BUILD_ID}"
echo "BUILD_ID: ${BUILD_ID}"
echo "Security: ${HAS_SECURITY}"
echo "Architecture: ${ARCHITECTURE}"

lib.jenkins.Messages.new(this).add('perf-test', "Performance tests for #${BUILD_ID}")
Expand Down Expand Up @@ -153,7 +151,6 @@ pipeline {

echo "BUNDLE_MANIFEST: ${BUNDLE_MANIFEST}"
echo "BUILD_ID: ${BUILD_ID}"
echo "BUILD_ID: ${BUILD_ID}"
echo "Architecture: ${ARCHITECTURE}"

lib.jenkins.Messages.new(this).add('perf-test', "Performance tests for #${BUILD_ID}")
Expand Down
11 changes: 6 additions & 5 deletions tests/jenkins/TestRunNonSecurityPerfTestScript.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,18 +33,19 @@ class TestRunSecurityPerfTestScript extends BuildPipelineTest {

@Test
public void testRunNonSecurityPerfTestScript_verifyPipeline() {
super.testPipeline("tests/jenkins/jobs/RunPerfTestScriptWithoutSecurity_Jenkinsfile")
super.testPipeline("jenkins/opensearch/perf-test.jenkinsfile",
"tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test.jenkinsfile")
}

@Test
void testRunNonSecurityPerfTestScript_verifyArtifactDownloads() {
runScript('tests/jenkins/jobs/RunPerfTestScriptWithoutSecurity_Jenkinsfile')
runScript("jenkins/opensearch/perf-test.jenkinsfile")

def curlCommands = getCommandExecutions('sh', 'curl').findAll {
shCommand -> shCommand.contains('curl')
}

assertThat(curlCommands.size(), equalTo(1))
assertThat(curlCommands.size(), equalTo(3))
assertThat(curlCommands, hasItem(
"curl test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml".toString()
))
Expand All @@ -61,7 +62,7 @@ class TestRunSecurityPerfTestScript extends BuildPipelineTest {

@Test
void testRunNonSecurityPerfTestScript_verifyPackageInstallation() {
runScript('tests/jenkins/jobs/RunPerfTestScriptWithoutSecurity_Jenkinsfile')
runScript("jenkins/opensearch/perf-test.jenkinsfile")

def npmCommands = getCommandExecutions('sh', 'npm').findAll {
shCommand -> shCommand.contains('npm')
Expand All @@ -79,7 +80,7 @@ class TestRunSecurityPerfTestScript extends BuildPipelineTest {

@Test
void testRunNonSecurityPerfTestScript_verifyScriptExecutions() {
runScript('tests/jenkins/jobs/RunPerfTestScriptWithoutSecurity_Jenkinsfile')
runScript("jenkins/opensearch/perf-test.jenkinsfile")

def testScriptCommands = getCommandExecutions('sh', './test.sh').findAll {
shCommand -> shCommand.contains('./test.sh')
Expand Down
11 changes: 6 additions & 5 deletions tests/jenkins/TestRunPerfTestScript.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,18 +33,19 @@ class TestRunPerfTestScript extends BuildPipelineTest {

@Test
public void testRunPerfTestScript_Pipeline() {
super.testPipeline("tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile")
super.testPipeline("jenkins/opensearch/perf-test.jenkinsfile",
"tests/jenkins/jenkinsjob-regression-files/opensearch/perf-test-with-security.jenkinsfile")
}

@Test
void testRunPerfTestScript_verifyArtifactDownloads() {
runScript('tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile')
runScript("jenkins/opensearch/perf-test.jenkinsfile")

def curlCommands = getCommandExecutions('sh', 'curl').findAll {
shCommand -> shCommand.contains('curl')
}

assertThat(curlCommands.size(), equalTo(2))
assertThat(curlCommands.size(), equalTo(4))
assertThat(curlCommands, hasItem(
"curl test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-bundle.yml".toString()
))
Expand All @@ -61,7 +62,7 @@ class TestRunPerfTestScript extends BuildPipelineTest {

@Test
void testRunPerfTestScript_verifyPackageInstallation() {
runScript('tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile')
runScript("jenkins/opensearch/perf-test.jenkinsfile")

def npmCommands = getCommandExecutions('sh', 'npm').findAll {
shCommand -> shCommand.contains('npm')
Expand All @@ -79,7 +80,7 @@ class TestRunPerfTestScript extends BuildPipelineTest {

@Test
void testRunPerfTestScript_verifyScriptExecutions() {
runScript('tests/jenkins/jobs/RunPerfTestScript_Jenkinsfile')
runScript("jenkins/opensearch/perf-test.jenkinsfile")

def testScriptCommands = getCommandExecutions('sh', './test.sh').findAll {
shCommand -> shCommand.contains('./test.sh')
Expand Down
108 changes: 73 additions & 35 deletions ...ns/jobs/RunPerfTestScript_Jenkinsfile.txt → ...h/perf-test-with-security.jenkinsfile.txt
View file
Original file line number Diff line number Diff line change
@@ -1,28 +1,39 @@
RunPerfTestScript_Jenkinsfile.run()
RunPerfTestScript_Jenkinsfile.legacySCM(groovy.lang.Closure)
RunPerfTestScript_Jenkinsfile.library({identifier=jenkins@20211118, retriever=null})
RunPerfTestScript_Jenkinsfile.pipeline(groovy.lang.Closure)
RunPerfTestScript_Jenkinsfile.timeout({time=10, unit=HOURS})
RunPerfTestScript_Jenkinsfile.echo(Executing on agent [label:none])
RunPerfTestScript_Jenkinsfile.stage(validate-and-set-parameters, groovy.lang.Closure)
RunPerfTestScript_Jenkinsfile.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]])
RunPerfTestScript_Jenkinsfile.script(groovy.lang.Closure)
RunPerfTestScript_Jenkinsfile.echo(Security: true)
RunPerfTestScript_Jenkinsfile.stage(test-with-security, groovy.lang.Closure)
RunPerfTestScript_Jenkinsfile.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]])
RunPerfTestScript_Jenkinsfile.script(groovy.lang.Closure)
RunPerfTestScript_Jenkinsfile.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-bundle.yml})
perf-test.run()
perf-test.legacySCM(groovy.lang.Closure)
perf-test.library({identifier=jenkins@20211118, retriever=null})
perf-test.pipeline(groovy.lang.Closure)
perf-test.timeout({time=10, unit=HOURS})
perf-test.echo(Executing on agent [label:none])
perf-test.stage(validate-and-set-parameters, groovy.lang.Closure)
perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]])
perf-test.script(groovy.lang.Closure)
perf-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-bundle.yml})
downloadBuildManifest.legacySCM(groovy.lang.Closure)
downloadBuildManifest.library({identifier=jenkins@20211123, retriever=null})
downloadBuildManifest.sh(curl test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-bundle.yml)
downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml})
BuildManifest.asBoolean()
RunPerfTestScript_Jenkinsfile.echo(BUNDLE_MANIFEST: tests/jenkins/data/opensearch-1.3.0-bundle.yml)
RunPerfTestScript_Jenkinsfile.echo(BUILD_ID: 1236)
RunPerfTestScript_Jenkinsfile.echo(BUILD_ID: 1236)
RunPerfTestScript_Jenkinsfile.echo(Security: true)
RunPerfTestScript_Jenkinsfile.echo(Architecture: x64)
RunPerfTestScript_Jenkinsfile.runPerfTestScript({bundleManifest=tests/jenkins/data/opensearch-1.3.0-bundle.yml, buildId=1236, architecture=x64, insecure=false, workload=nyc_taxis, testIterations=1, warmupIterations=1})
perf-test.echo(Security: true)
BuildManifest.getArtifactBuildId()
BuildManifest.getArtifactArchitecture()
perf-test.echo(Security: true)
perf-test.stage(test-with-security, groovy.lang.Closure)
perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]])
perf-test.script(groovy.lang.Closure)
perf-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-bundle.yml})
downloadBuildManifest.legacySCM(groovy.lang.Closure)
downloadBuildManifest.library({identifier=jenkins@20211123, retriever=null})
downloadBuildManifest.sh(curl test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-bundle.yml)
downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml})
BuildManifest.asBoolean()
perf-test.echo(BUNDLE_MANIFEST: tests/jenkins/data/opensearch-1.3.0-bundle.yml)
perf-test.echo(BUILD_ID: 1236)
perf-test.echo(Architecture: x64)
Messages.asBoolean()
Messages.add(perf-test, Performance tests for #1236)
perf-test.writeFile({file=messages/perf-test.msg, text=Performance tests for #1236})
perf-test.stash({includes=messages/*, name=messages-perf-test})
perf-test.runPerfTestScript({bundleManifest=tests/jenkins/data/opensearch-1.3.0-bundle.yml, buildId=1236, architecture=x64, insecure=false, workload=nyc_taxis, testIterations=1, warmupIterations=1})
runPerfTestScript.legacySCM(groovy.lang.Closure)
runPerfTestScript.library({identifier=jenkins@20211123, retriever=null})
runPerfTestScript.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml})
Expand All @@ -42,8 +53,12 @@
runPerfTestScript.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
runPerfTestScript.s3Download({file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true})
runPerfTestScript.sh(./test.sh perf-test --stack test-single-security-1236-x64 --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config config.yml --workload nyc_taxis --test-iters 1 --warmup-iters 1)
RunPerfTestScript_Jenkinsfile.script(groovy.lang.Closure)
RunPerfTestScript_Jenkinsfile.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-1.3.0-bundle.yml, jobName=perf-test, buildNumber=1236})
Messages.asBoolean()
Messages.add(perf-test, Performance tests with security for 1236 completed)
perf-test.writeFile({file=messages/perf-test.msg, text=Performance tests with security for 1236 completed})
perf-test.stash({includes=messages/*, name=messages-perf-test})
perf-test.script(groovy.lang.Closure)
perf-test.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-1.3.0-bundle.yml, jobName=perf-test, buildNumber=1236})
uploadTestResults.legacySCM(groovy.lang.Closure)
uploadTestResults.library({identifier=jenkins@20211123, retriever=null})
uploadTestResults.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml})
Expand All @@ -57,22 +72,25 @@
Messages.add(test_stage, test://artifact.url/perf-test/1.3.0/1236/linux/x64/test-results/)
uploadTestResults.writeFile({file=messages/test_stage.msg, text=test://artifact.url/perf-test/1.3.0/1236/linux/x64/test-results/})
uploadTestResults.stash({includes=messages/*, name=messages-test_stage})
RunPerfTestScript_Jenkinsfile.postCleanup()
perf-test.postCleanup()
postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true})
RunPerfTestScript_Jenkinsfile.stage(test-without-security, groovy.lang.Closure)
RunPerfTestScript_Jenkinsfile.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]])
RunPerfTestScript_Jenkinsfile.script(groovy.lang.Closure)
RunPerfTestScript_Jenkinsfile.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-bundle.yml})
perf-test.stage(test-without-security, groovy.lang.Closure)
perf-test.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]])
perf-test.script(groovy.lang.Closure)
perf-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-bundle.yml})
downloadBuildManifest.legacySCM(groovy.lang.Closure)
downloadBuildManifest.library({identifier=jenkins@20211123, retriever=null})
downloadBuildManifest.sh(curl test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-bundle.yml)
downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml})
BuildManifest.asBoolean()
RunPerfTestScript_Jenkinsfile.echo(BUNDLE_MANIFEST: tests/jenkins/data/opensearch-1.3.0-bundle.yml)
RunPerfTestScript_Jenkinsfile.echo(BUILD_ID: 1236)
RunPerfTestScript_Jenkinsfile.echo(BUILD_ID: 1236)
RunPerfTestScript_Jenkinsfile.echo(Architecture: x64)
RunPerfTestScript_Jenkinsfile.runPerfTestScript({bundleManifest=tests/jenkins/data/opensearch-1.3.0-bundle.yml, buildId=1236, architecture=x64, insecure=true, workload=nyc_taxis, testIterations=1, warmupIterations=1})
perf-test.echo(BUNDLE_MANIFEST: tests/jenkins/data/opensearch-1.3.0-bundle.yml)
perf-test.echo(BUILD_ID: 1236)
perf-test.echo(Architecture: x64)
Messages.asBoolean()
Messages.add(perf-test, Performance tests for #1236)
perf-test.writeFile({file=messages/perf-test.msg, text=Performance tests for #1236})
perf-test.stash({includes=messages/*, name=messages-perf-test})
perf-test.runPerfTestScript({bundleManifest=tests/jenkins/data/opensearch-1.3.0-bundle.yml, buildId=1236, architecture=x64, insecure=true, workload=nyc_taxis, testIterations=1, warmupIterations=1})
runPerfTestScript.legacySCM(groovy.lang.Closure)
runPerfTestScript.library({identifier=jenkins@20211123, retriever=null})
runPerfTestScript.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml})
Expand All @@ -92,8 +110,12 @@
runPerfTestScript.withAWS({role=opensearch-test, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
runPerfTestScript.s3Download({file=config.yml, bucket=test_bucket, path=test_config/config.yml, force=true})
runPerfTestScript.sh(./test.sh perf-test --stack test-single-1236-x64 --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config config.yml --force-insecure-mode --workload nyc_taxis --test-iters 1 --warmup-iters 1)
RunPerfTestScript_Jenkinsfile.script(groovy.lang.Closure)
RunPerfTestScript_Jenkinsfile.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-1.3.0-bundle.yml, jobName=perf-test, buildNumber=1236})
Messages.asBoolean()
Messages.add(perf-test, Performance tests without security for 1236 completed)
perf-test.writeFile({file=messages/perf-test.msg, text=Performance tests without security for 1236 completed})
perf-test.stash({includes=messages/*, name=messages-perf-test})
perf-test.script(groovy.lang.Closure)
perf-test.uploadTestResults({buildManifestFileName=tests/jenkins/data/opensearch-1.3.0-bundle.yml, jobName=perf-test, buildNumber=1236})
uploadTestResults.legacySCM(groovy.lang.Closure)
uploadTestResults.library({identifier=jenkins@20211123, retriever=null})
uploadTestResults.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml})
Expand All @@ -107,5 +129,21 @@
Messages.add(test_stage, test://artifact.url/perf-test/1.3.0/1236/linux/x64/test-results/)
uploadTestResults.writeFile({file=messages/test_stage.msg, text=test://artifact.url/perf-test/1.3.0/1236/linux/x64/test-results/})
uploadTestResults.stash({includes=messages/*, name=messages-test_stage})
RunPerfTestScript_Jenkinsfile.postCleanup()
perf-test.postCleanup()
postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true})
perf-test.node(Jenkins-Agent-al2-x64-c54xlarge-Docker-Host, groovy.lang.Closure)
perf-test.script(groovy.lang.Closure)
Messages.asBoolean()
Messages.get([perf-test])
perf-test.unstash({name=messages-perf-test})
perf-test.findFiles({excludes=, glob=messages/*})
perf-test.dir(messages, groovy.lang.Closure)
perf-test.deleteDir()
perf-test.publishNotification({icon=:white_check_mark:, message=Performance Tests Successful, extra=, credentialsId=INTEG_TEST_WEBHOOK})
publishNotification.string({credentialsId=INTEG_TEST_WEBHOOK, variable=WEBHOOK_URL})
publishNotification.withCredentials([WEBHOOK_URL], groovy.lang.Closure)
publishNotification.sh(curl -XPOST --header "Content-Type: application/json" --data '{"result_text":":white_check_mark: perf-test [1236] Performance Tests Successful
Build: test://artifact.url
"}' "WEBHOOK_URL")
perf-test.postCleanup()
postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true})
Loading

0 comments on commit 3f98feb

Please sign in to comment.