From cd1b20bfeda82d3d7b4d88881d36a4e20f9b447b Mon Sep 17 00:00:00 2001
From: Mike Place <mike.place@elastic.co>
Date: Wed, 21 Oct 2020 07:45:48 +0200
Subject: [PATCH 01/37] Skeleton of perf Jenkinsfile

---
 .ci/load/Jenkinsfile | 65 ++++++++++++++++++++++++++++++++++++++++++++
 .ci/load/locust.py   | 21 ++++++++++++++
 2 files changed, 86 insertions(+)
 create mode 100644 .ci/load/Jenkinsfile
 create mode 100644 .ci/load/locust.py

diff --git a/.ci/load/Jenkinsfile b/.ci/load/Jenkinsfile
new file mode 100644
index 0000000000..5ba96353fc
--- /dev/null
+++ b/.ci/load/Jenkinsfile
@@ -0,0 +1,65 @@
+pipeline {
+    agent any
+    environment {
+        REPO = 'apm-agent-java'
+        APP = 'spring-petclinic'
+        APP_BASE_DIR = "src/${APP}"
+        // BASE_DIR = "src/github.com/elastic/${env.REPO}"
+        // DOCKERHUB_SECRET = 'secret/apm-team/ci/elastic-observability-dockerhub'
+        // ELASTIC_DOCKER_SECRET = 'secret/apm-team/ci/docker-registry/prod'
+        // NEXUS_SECRET = 'secret/apm-team/ci/nexus'
+        // MAVEN_CONFIG = '-B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -Dmaven.repo.local=.m2'
+        // HOME = "${env.WORKSPACE}"
+        // JAVA_HOME = "${env.HUDSON_HOME}/.java/java10"
+        // PATH = "${env.JAVA_HOME}/bin:${env.PATH}"
+    }
+    options {
+        timeout(time: 3, unit: 'HOURS')
+        buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30'))
+        timestamps()
+        ansiColor('xterm')
+        durabilityHint('PERFORMANCE_OPTIMIZED')
+
+    }
+    parameters {
+    // TODO: Allow for SHA instead?
+    string(name: 'agent_version', defaultValue: 'v1.9.0', description: 'Version of agent. Should correspond to tag, e.g. `v1.9.0`.')
+    // TODO: Drop-down of JVM versions instead
+    string(name: 'jvm_version', defaultValue: '9.0.4', description: 'Version of JVM.')
+    string(name: 'concurrent_requests', defaultValue: '100', description: 'The number of concurrent requests to test with.')
+    string(name: 'duration', defaultValue: '10', description: 'Test duration in minutes. Max: 280') // 300 min is the overall timeout so 280 is probably fine.
+    string(name: 'num_of_runs', defaultValue: '1', description: 'Number of test runs to execute.')
+  }
+    stages {
+        stage('Prepare test application') {
+            steps {
+                gitCheckout(
+                    basedir: "${APP_BASE_DIR}",
+                    branch: 'main',
+                    repo: "https://github.com/spring-projects/${APP}.git",
+                    credentialsId: 'f6c7695a-671e-4f4f-a331-acdce44ff9ba',
+                    shallow: false
+                )
+            }
+        }
+        stage('Start test application') {
+            steps {
+                dir("${APP_BASE_DIR}"){
+                    sh(script: "./mvnw spring-boot:run")
+                }
+                echo 'Implement me!'
+                echo 'I will need to use the options passed in!'
+            }
+        }
+        stage('Download load-generation container') {
+            steps {
+                echo 'Implement me!'
+            }
+        }
+        stage('Start load generation') {
+            steps {
+                echo 'Implement me!'
+            }
+        }
+    }
+}
diff --git a/.ci/load/locust.py b/.ci/load/locust.py
new file mode 100644
index 0000000000..1ba93edfb3
--- /dev/null
+++ b/.ci/load/locust.py
@@ -0,0 +1,21 @@
+import time
+from locust import HttpUser, task, between
+
+
+class QuickstartUser(HttpUser):
+    wait_time = between(1, 2)
+
+    @task
+    def index_page(self):
+        self.client.get("/hello")
+        self.client.get("/world")
+
+    @task(3)
+    def view_item(self):
+        for item_id in range(10):
+            self.client.get(f"/item?id={item_id}", name="/item")
+            time.sleep(1)
+
+    def on_start(self):
+        self.client.post("/login", json={"username":"foo", "password":"bar"})
+

From d3e93d2306f8bdbd1ebb17455eebbcd7542bd37e Mon Sep 17 00:00:00 2001
From: Mike Place <mike.place@elastic.co>
Date: Thu, 22 Oct 2020 16:10:11 +0200
Subject: [PATCH 02/37] load generation wait script

---
 .ci/load/Jenkinsfile                          | 49 ++++++++--------
 .ci/load/scripts/load_agent.sh                | 56 +++++++++++++++++++
 .ci/load/{locust.py => scripts/locustfile.py} |  0
 .ci/load/scripts/tests/simulate_app_start.sh  | 12 ++++
 4 files changed, 93 insertions(+), 24 deletions(-)
 create mode 100755 .ci/load/scripts/load_agent.sh
 rename .ci/load/{locust.py => scripts/locustfile.py} (100%)
 create mode 100755 .ci/load/scripts/tests/simulate_app_start.sh

diff --git a/.ci/load/Jenkinsfile b/.ci/load/Jenkinsfile
index 5ba96353fc..88ec44a661 100644
--- a/.ci/load/Jenkinsfile
+++ b/.ci/load/Jenkinsfile
@@ -31,34 +31,35 @@ pipeline {
     string(name: 'num_of_runs', defaultValue: '1', description: 'Number of test runs to execute.')
   }
     stages {
-        stage('Prepare test application') {
+        stage('Pre-flight'){
             steps {
-                gitCheckout(
-                    basedir: "${APP_BASE_DIR}",
-                    branch: 'main',
-                    repo: "https://github.com/spring-projects/${APP}.git",
-                    credentialsId: 'f6c7695a-671e-4f4f-a331-acdce44ff9ba',
-                    shallow: false
-                )
+                echo 'Pre-flight implement me!'
             }
         }
-        stage('Start test application') {
-            steps {
-                dir("${APP_BASE_DIR}"){
-                    sh(script: "./mvnw spring-boot:run")
+        stage('Load test') {
+            parallel {
+                stage('Start load generation') {
+                    agent { label 'linux && immutable' }
+                    steps {
+                        echo 'Install load generation tooling'
+                        echo 'LOAD: Implement me!'
+                    }
+                }
+                stage('Start application') {
+                    steps {
+                        echo 'APP: Implement me!'
+                        gitCheckout(
+                            basedir: "${APP_BASE_DIR}",
+                            branch: 'main',
+                            repo: "https://github.com/spring-projects/${APP}.git",
+                            credentialsId: 'f6c7695a-671e-4f4f-a331-acdce44ff9ba',
+                            shallow: false
+                        )
+                        dir("${APP_BASE_DIR}"){
+                            sh(script: "./mvnw spring-boot:run")
+                        }
+                    }
                 }
-                echo 'Implement me!'
-                echo 'I will need to use the options passed in!'
-            }
-        }
-        stage('Download load-generation container') {
-            steps {
-                echo 'Implement me!'
-            }
-        }
-        stage('Start load generation') {
-            steps {
-                echo 'Implement me!'
             }
         }
     }
diff --git a/.ci/load/scripts/load_agent.sh b/.ci/load/scripts/load_agent.sh
new file mode 100755
index 0000000000..df42709b6f
--- /dev/null
+++ b/.ci/load/scripts/load_agent.sh
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+
+set -exuo pipefail 
+
+POLL_FREQ=1
+
+LOCUST_LOCUSTFILE="../locust.py"
+LOCUST_PRINT_STATS=1
+
+# 1. Loop until we get a signal that the application is ready.
+
+function appIsReady() {
+
+    curl -s -X POST -H "Content-Type: application/json" -d \
+    "{\"app_token\": \""$APP_TOKEN"\", \"session_token\": \""$SESSION_TOKEN"\"}" $ORCH_URL/api/poll|jq
+
+    curl -s -X POST -H "Content-Type: application/json" -d \
+    "{\"app_token\": \""$APP_TOKEN"\", \"session_token\": \""$SESSION_TOKEN"\"}" $ORCH_URL/api/poll | \
+    jq -e '.services.application.state == "ready"'
+
+
+}
+
+function waitForApp() {
+    while :
+    do
+        if appIsReady; then 
+            break 
+        fi
+        sleep $POLL_FREQ;
+    done
+}
+
+function buildArgs() {
+    export LOCUST_HOST=$(curl -s -X POST -H "Content-Type: application/json" -d \
+    "{\"app_token\": \""$APP_TOKEN"\", \"session_token\": \""$SESSION_TOKEN"\"}" $ORCH_URL/api/poll | jq '.services.application.'})
+
+    export LOCUST_RUN_TIME=30
+}
+
+function startLoad() {
+    curl -s -X POST -H "Content-Type: application/json" -d \
+    "{\"app_token\": \""$APP_TOKEN"\", \"session_token\": \""$SESSION_TOKEN"\", \"service\": \"load_generation\"}" $ORCH_URL/api/ready
+
+    docker run -p 8089:8089 -v ${PWD}/.ci/load/scripts:/locust locustio/locust -f /locust/locustfile.py
+}
+
+function stopLoad() {
+    # Stop the load test
+    docker ps|egrep locust|awk '{print $1}'|xargs docker kill
+}
+
+
+waitForApp
+buildArgs
+startLoad
diff --git a/.ci/load/locust.py b/.ci/load/scripts/locustfile.py
similarity index 100%
rename from .ci/load/locust.py
rename to .ci/load/scripts/locustfile.py
diff --git a/.ci/load/scripts/tests/simulate_app_start.sh b/.ci/load/scripts/tests/simulate_app_start.sh
new file mode 100755
index 0000000000..37485157d2
--- /dev/null
+++ b/.ci/load/scripts/tests/simulate_app_start.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -exuo pipefail 
+
+POLL_FREQ=1
+
+function startApp() {
+    curl -s -X POST -H "Content-Type: application/json" -d \
+    "{\"app_token\": \""$APP_TOKEN"\", \"session_token\": \""$SESSION_TOKEN"\", \"service\": \"application\", \"hostname\": \"test_app\", \"port\": \"999\"}" $ORCH_URL/api/ready 
+}
+
+startApp
\ No newline at end of file

From 0521d524ff15327ccc7707eb8f37b061dfa91400 Mon Sep 17 00:00:00 2001
From: Mike Place <mike.place@elastic.co>
Date: Mon, 26 Oct 2020 16:50:01 +0100
Subject: [PATCH 03/37] More orch

---
 .ci/load/Jenkinsfile                         | 31 ++++++------
 .ci/load/scripts/app.sh                      | 53 ++++++++++++++++++++
 .ci/load/scripts/load_agent.sh               | 19 ++++---
 .ci/load/scripts/start.sh                    | 14 ++++++
 .ci/load/scripts/tests/simulate_app_start.sh |  7 ++-
 .ci/load/scripts/tests/simulate_register.sh  | 14 ++++++
 6 files changed, 114 insertions(+), 24 deletions(-)
 create mode 100755 .ci/load/scripts/app.sh
 create mode 100755 .ci/load/scripts/start.sh
 create mode 100755 .ci/load/scripts/tests/simulate_register.sh

diff --git a/.ci/load/Jenkinsfile b/.ci/load/Jenkinsfile
index 88ec44a661..01a78a7546 100644
--- a/.ci/load/Jenkinsfile
+++ b/.ci/load/Jenkinsfile
@@ -4,14 +4,7 @@ pipeline {
         REPO = 'apm-agent-java'
         APP = 'spring-petclinic'
         APP_BASE_DIR = "src/${APP}"
-        // BASE_DIR = "src/github.com/elastic/${env.REPO}"
-        // DOCKERHUB_SECRET = 'secret/apm-team/ci/elastic-observability-dockerhub'
-        // ELASTIC_DOCKER_SECRET = 'secret/apm-team/ci/docker-registry/prod'
-        // NEXUS_SECRET = 'secret/apm-team/ci/nexus'
-        // MAVEN_CONFIG = '-B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -Dmaven.repo.local=.m2'
-        // HOME = "${env.WORKSPACE}"
-        // JAVA_HOME = "${env.HUDSON_HOME}/.java/java10"
-        // PATH = "${env.JAVA_HOME}/bin:${env.PATH}"
+        ORCH_URL='localhost:8000' // FIXME: For development only.
     }
     options {
         timeout(time: 3, unit: 'HOURS')
@@ -27,22 +20,30 @@ pipeline {
     // TODO: Drop-down of JVM versions instead
     string(name: 'jvm_version', defaultValue: '9.0.4', description: 'Version of JVM.')
     string(name: 'concurrent_requests', defaultValue: '100', description: 'The number of concurrent requests to test with.')
-    string(name: 'duration', defaultValue: '10', description: 'Test duration in minutes. Max: 280') // 300 min is the overall timeout so 280 is probably fine.
+    // 300 min is the overall timeout so 280 is probably fine.
+    string(name: 'duration', defaultValue: '10', description: 'Test duration in minutes. Max: 280')
     string(name: 'num_of_runs', defaultValue: '1', description: 'Number of test runs to execute.')
-  }
+    }
+
     stages {
         stage('Pre-flight'){
             steps {
-                echo 'Pre-flight implement me!'
-            }
+                echo 'Getting authentication information from Vault'
+                withSecretVault(secret: 'secret/apm-team/ci/bandstand', user_var_name: 'APP_TOKEN_TYPE', pass_var_name: 'APP_TOKEN'){
+                    script {
+                        env.SESSION_TOKEN = sh(script: ".ci/load/scripts/start.sh")
+                    }
+                }
+            }    
         }
         stage('Load test') {
             parallel {
                 stage('Start load generation') {
                     agent { label 'linux && immutable' }
                     steps {
-                        echo 'Install load generation tooling'
-                        echo 'LOAD: Implement me!'
+                            withSecretVault(secret: 'secret/apm-team/ci/bandstand', user_var_name: 'APP_TOKEN_TYPE', pass_var_name: 'APP_TOKEN'){
+                                sh(script: ".ci/load/scripts/load_agent.sh")
+                            }
                     }
                 }
                 stage('Start application') {
@@ -63,4 +64,4 @@ pipeline {
             }
         }
     }
-}
+}
\ No newline at end of file
diff --git a/.ci/load/scripts/app.sh b/.ci/load/scripts/app.sh
new file mode 100755
index 0000000000..079fb5b27b
--- /dev/null
+++ b/.ci/load/scripts/app.sh
@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+
+set -exuo pipefail 
+
+POLL_FREQ=1
+
+APP_PORT=999
+
+
+function sendAppReady() {
+    curl -s -X POST -H "Content-Type: application/json" -d \
+    "{\"app_token\": \""$APP_TOKEN"\", \
+    \"session_token\": \""$SESSION_TOKEN"\", \
+    \"service\": \"load_generation\", \
+    \"hostname\": \""$(hostname)"\", \
+    \"port\": \"999\"}" \
+    $ORCH_URL/api/ready 
+
+}
+
+function waitForApp() {
+    while :
+    do
+        if appIsReady; then 
+            break 
+        fi
+        sleep $POLL_FREQ;
+    done
+}
+
+function buildArgs() {
+    export LOCUST_HOST=$(curl -s -X POST -H "Content-Type: application/json" -d \
+    "{\"app_token\": \""$APP_TOKEN"\", \"session_token\": \""$SESSION_TOKEN"\"}" $ORCH_URL/api/poll | jq '.services.application.'})
+
+    export LOCUST_RUN_TIME=30
+}
+
+function startLoad() {
+    curl -s -X POST -H "Content-Type: application/json" -d \
+    "{\"app_token\": \""$APP_TOKEN"\", \"session_token\": \""$SESSION_TOKEN"\", \"service\": \"load_generation\"}" $ORCH_URL/api/ready
+
+    docker run -p 8089:8089 -v ${PWD}/.ci/load/scripts:/locust locustio/locust -f /locust/locustfile.py
+}
+
+function stopLoad() {
+    # Stop the load test
+    docker ps|egrep locust|awk '{print $1}'|xargs docker kill
+}
+
+sendAppReady
+# waitForApp
+# buildArgs
+# startLoad
\ No newline at end of file
diff --git a/.ci/load/scripts/load_agent.sh b/.ci/load/scripts/load_agent.sh
index df42709b6f..5267442701 100755
--- a/.ci/load/scripts/load_agent.sh
+++ b/.ci/load/scripts/load_agent.sh
@@ -10,15 +10,9 @@ LOCUST_PRINT_STATS=1
 # 1. Loop until we get a signal that the application is ready.
 
 function appIsReady() {
-
-    curl -s -X POST -H "Content-Type: application/json" -d \
-    "{\"app_token\": \""$APP_TOKEN"\", \"session_token\": \""$SESSION_TOKEN"\"}" $ORCH_URL/api/poll|jq
-
     curl -s -X POST -H "Content-Type: application/json" -d \
     "{\"app_token\": \""$APP_TOKEN"\", \"session_token\": \""$SESSION_TOKEN"\"}" $ORCH_URL/api/poll | \
     jq -e '.services.application.state == "ready"'
-
-
 }
 
 function waitForApp() {
@@ -33,14 +27,23 @@ function waitForApp() {
 
 function buildArgs() {
     export LOCUST_HOST=$(curl -s -X POST -H "Content-Type: application/json" -d \
-    "{\"app_token\": \""$APP_TOKEN"\", \"session_token\": \""$SESSION_TOKEN"\"}" $ORCH_URL/api/poll | jq '.services.application.'})
+    "{\"app_token\": \
+    \""$APP_TOKEN"\", \
+    \"session_token\": \""$SESSION_TOKEN"\"}" \
+    $ORCH_URL/api/poll | \
+    jq '.services.application.'})
 
     export LOCUST_RUN_TIME=30
 }
 
 function startLoad() {
     curl -s -X POST -H "Content-Type: application/json" -d \
-    "{\"app_token\": \""$APP_TOKEN"\", \"session_token\": \""$SESSION_TOKEN"\", \"service\": \"load_generation\"}" $ORCH_URL/api/ready
+    "{\"app_token\": \""$APP_TOKEN"\", \
+    \"session_token\": \""$SESSION_TOKEN"\", \
+    \"service\": \"load_generation\", \
+    \"hostname\": \"test_app\", \
+    \"port\": \"999\"}" \
+    $ORCH_URL/api/ready 
 
     docker run -p 8089:8089 -v ${PWD}/.ci/load/scripts:/locust locustio/locust -f /locust/locustfile.py
 }
diff --git a/.ci/load/scripts/start.sh b/.ci/load/scripts/start.sh
new file mode 100755
index 0000000000..a5953c982c
--- /dev/null
+++ b/.ci/load/scripts/start.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+# set -exuo pipefail 
+
+function registerSession() {
+    curl -s -X POST -H "Content-Type: application/json" -d \
+    "{\"app_token\": \""$APP_TOKEN"\", \
+    \"service\": \"application\", \
+    \"hostname\": \"test_app\", \
+    \"port\": \"999\"}" \
+    $ORCH_URL/api/register | jq -Mr '.session_created.session'
+}
+
+registerSession
\ No newline at end of file
diff --git a/.ci/load/scripts/tests/simulate_app_start.sh b/.ci/load/scripts/tests/simulate_app_start.sh
index 37485157d2..d443bf4810 100755
--- a/.ci/load/scripts/tests/simulate_app_start.sh
+++ b/.ci/load/scripts/tests/simulate_app_start.sh
@@ -6,7 +6,12 @@ POLL_FREQ=1
 
 function startApp() {
     curl -s -X POST -H "Content-Type: application/json" -d \
-    "{\"app_token\": \""$APP_TOKEN"\", \"session_token\": \""$SESSION_TOKEN"\", \"service\": \"application\", \"hostname\": \"test_app\", \"port\": \"999\"}" $ORCH_URL/api/ready 
+    "{\"app_token\": \""$APP_TOKEN"\", \
+    \"session_token\": \""$SESSION_TOKEN"\", \
+    \"service\": \"application\", \
+    \"hostname\": \"test_app\", \
+    \"port\": \"999\"}" \
+    $ORCH_URL/api/ready 
 }
 
 startApp
\ No newline at end of file
diff --git a/.ci/load/scripts/tests/simulate_register.sh b/.ci/load/scripts/tests/simulate_register.sh
new file mode 100755
index 0000000000..d084e8a249
--- /dev/null
+++ b/.ci/load/scripts/tests/simulate_register.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -exuo pipefail 
+
+function registerSession() {
+    curl -s -X POST -H "Content-Type: application/json" -d \
+    "{\"app_token\": \""$APP_TOKEN"\", \
+    \"service\": \"application\", \
+    \"hostname\": \"test_app\", \
+    \"port\": \"999\"}" \
+    $ORCH_URL/api/register | jq '.session_created.session'
+}
+
+registerSession
\ No newline at end of file

From 6c979b9a91270bb4cd119fe8c76e1b997898f942 Mon Sep 17 00:00:00 2001
From: Mike Place <mike.place@elastic.co>
Date: Tue, 27 Oct 2020 15:55:23 +0100
Subject: [PATCH 04/37] Working PoC

---
 .ci/load/Jenkinsfile           | 14 +++++--
 .ci/load/scripts/app.sh        | 73 ++++++++++++++++++++++++++--------
 .ci/load/scripts/load_agent.sh | 41 +++++++++++++------
 .ci/load/scripts/start.sh      | 16 +++-----
 4 files changed, 100 insertions(+), 44 deletions(-)

diff --git a/.ci/load/Jenkinsfile b/.ci/load/Jenkinsfile
index 01a78a7546..16fc945591 100644
--- a/.ci/load/Jenkinsfile
+++ b/.ci/load/Jenkinsfile
@@ -4,7 +4,7 @@ pipeline {
         REPO = 'apm-agent-java'
         APP = 'spring-petclinic'
         APP_BASE_DIR = "src/${APP}"
-        ORCH_URL='localhost:8000' // FIXME: For development only.
+        ORCH_URL='10.0.2.2:8000' // FIXME: For development only.
     }
     options {
         timeout(time: 3, unit: 'HOURS')
@@ -31,7 +31,9 @@ pipeline {
                 echo 'Getting authentication information from Vault'
                 withSecretVault(secret: 'secret/apm-team/ci/bandstand', user_var_name: 'APP_TOKEN_TYPE', pass_var_name: 'APP_TOKEN'){
                     script {
-                        env.SESSION_TOKEN = sh(script: ".ci/load/scripts/start.sh")
+                        sh(script: ".ci/load/scripts/start.sh", returnStdout: true)
+                        env.LOCUST_RUN_TIME = params.duration
+                        env.SESSION_TOKEN = sh(script: ".ci/load/scripts/start.sh", returnStdout: true).trim()
                     }
                 }
             }    
@@ -48,7 +50,6 @@ pipeline {
                 }
                 stage('Start application') {
                     steps {
-                        echo 'APP: Implement me!'
                         gitCheckout(
                             basedir: "${APP_BASE_DIR}",
                             branch: 'main',
@@ -57,7 +58,12 @@ pipeline {
                             shallow: false
                         )
                         dir("${APP_BASE_DIR}"){
-                            sh(script: "./mvnw spring-boot:run")
+                            // Launch app in background
+                            sh(script: "./mvnw spring-boot:run &")
+                        }
+                        // Foreground the orchestrator script for execution control
+                        withSecretVault(secret: 'secret/apm-team/ci/bandstand', user_var_name: 'APP_TOKEN_TYPE', pass_var_name: 'APP_TOKEN'){
+                            sh(script: ".ci/load/scripts/app.sh")
                         }
                     }
                 }
diff --git a/.ci/load/scripts/app.sh b/.ci/load/scripts/app.sh
index 079fb5b27b..ee64a270c2 100755
--- a/.ci/load/scripts/app.sh
+++ b/.ci/load/scripts/app.sh
@@ -4,21 +4,27 @@ set -exuo pipefail
 
 POLL_FREQ=1
 
-APP_PORT=999
+APP_PORT=8080
 
+function appIsReady() {
+    # Poll the app until it is ready
+    curl -Is http://localhost:$APP_PORT/| head -1|egrep 200
+}
 
 function sendAppReady() {
     curl -s -X POST -H "Content-Type: application/json" -d \
     "{\"app_token\": \""$APP_TOKEN"\", \
     \"session_token\": \""$SESSION_TOKEN"\", \
-    \"service\": \"load_generation\", \
+    \"service\": \"application\", \
     \"hostname\": \""$(hostname)"\", \
-    \"port\": \"999\"}" \
+    \"port\": \"8080\"}" \
     $ORCH_URL/api/ready 
-
+    
+    
 }
 
 function waitForApp() {
+    # Wait for the load generation to finish before we kill the app
     while :
     do
         if appIsReady; then 
@@ -28,26 +34,59 @@ function waitForApp() {
     done
 }
 
-function buildArgs() {
-    export LOCUST_HOST=$(curl -s -X POST -H "Content-Type: application/json" -d \
-    "{\"app_token\": \""$APP_TOKEN"\", \"session_token\": \""$SESSION_TOKEN"\"}" $ORCH_URL/api/poll | jq '.services.application.'})
+function waitForLoad() {
+    # Wait for the load generation to finish before we kill the app
+    while :
+    do
+        if checkLoadGen; then 
+            break 
+        fi
+        sleep $POLL_FREQ;
+    done
+}
 
-    export LOCUST_RUN_TIME=30
+function waitForLoadFinish() {
+    # Wait for the load generation to finish before we kill the app
+    while :
+    do
+        if checkLoadGenFinish; then 
+            break 
+        fi
+        sleep $POLL_FREQ;
+    done
 }
 
-function startLoad() {
+
+function checkLoadGen(){
+    # Check to see if the load generation piece is sending requests
     curl -s -X POST -H "Content-Type: application/json" -d \
-    "{\"app_token\": \""$APP_TOKEN"\", \"session_token\": \""$SESSION_TOKEN"\", \"service\": \"load_generation\"}" $ORCH_URL/api/ready
+    "{\"app_token\": \""$APP_TOKEN"\", \
+    \"session_token\": \""$SESSION_TOKEN"\", \
+    \"service\": \"load_generation\", \
+    \"hostname\": \"test_app\", \
+    \"port\": \"8080\"}" \
+    $ORCH_URL/api/poll | jq '.services.load_generation.state' | egrep 'ready'
+}
 
-    docker run -p 8089:8089 -v ${PWD}/.ci/load/scripts:/locust locustio/locust -f /locust/locustfile.py
+
+function checkLoadGenFinish(){
+    # Check to see if the load generation piece is sending requests
+    curl -s -X POST -H "Content-Type: application/json" -d \
+    "{\"app_token\": \""$APP_TOKEN"\", \
+    \"session_token\": \""$SESSION_TOKEN"\", \
+    \"service\": \"load_generation\", \
+    \"hostname\": \"test_app\", \
+    \"port\": \"8080\"}" \
+    $ORCH_URL/api/poll | jq '.services.load_generation.state' | egrep 'stopped'
 }
 
-function stopLoad() {
-    # Stop the load test
-    docker ps|egrep locust|awk '{print $1}'|xargs docker kill
+
+function stopApp() {
+    ps -ef|egrep spring-boot:run|egrep java|awk '{print $2}'|xargs kill
 }
 
+waitForApp
 sendAppReady
-# waitForApp
-# buildArgs
-# startLoad
\ No newline at end of file
+waitForLoad
+waitForLoadFinish
+stopApp
diff --git a/.ci/load/scripts/load_agent.sh b/.ci/load/scripts/load_agent.sh
index 5267442701..a661292462 100755
--- a/.ci/load/scripts/load_agent.sh
+++ b/.ci/load/scripts/load_agent.sh
@@ -26,14 +26,22 @@ function waitForApp() {
 }
 
 function buildArgs() {
-    export LOCUST_HOST=$(curl -s -X POST -H "Content-Type: application/json" -d \
-    "{\"app_token\": \
-    \""$APP_TOKEN"\", \
-    \"session_token\": \""$SESSION_TOKEN"\"}" \
-    $ORCH_URL/api/poll | \
-    jq '.services.application.'})
-
-    export LOCUST_RUN_TIME=30
+    LOCUST_HOSTNAME="$(curl -s -X POST -H "Content-Type: application/json" -d \
+        "{\"app_token\": \
+        \""$APP_TOKEN"\", \
+        \"session_token\": \""$SESSION_TOKEN"\"}" \
+        $ORCH_URL/api/poll | \
+        jq '.services.application.hostname')"
+    
+    LOCUST_PORT="$(curl -s -X POST -H "Content-Type: application/json" -d \
+        "{\"app_token\": \
+        \""$APP_TOKEN"\", \
+        \"session_token\": \""$SESSION_TOKEN"\"}" \
+        $ORCH_URL/api/poll | \
+        jq '.services.application.port')"
+    # FIXME temporary value
+    export LOCUST_HOST=http://$(echo $LOCUST_HOSTNAME|sed 's/"//g'):$(echo $LOCUST_PORT|sed 's/"//g')
+    export LOCUST_RUN_TIME=30s
 }
 
 function startLoad() {
@@ -42,18 +50,25 @@ function startLoad() {
     \"session_token\": \""$SESSION_TOKEN"\", \
     \"service\": \"load_generation\", \
     \"hostname\": \"test_app\", \
-    \"port\": \"999\"}" \
+    \"port\": \"8080\"}" \
     $ORCH_URL/api/ready 
-
-    docker run -p 8089:8089 -v ${PWD}/.ci/load/scripts:/locust locustio/locust -f /locust/locustfile.py
+    # FIXME put number of concurrent users in
+    docker run -e "LOCUST_HOST=$LOCUST_HOST" -e "LOCUST_RUN_TIME=$LOCUST_RUN_TIME" -p 8089:8089 -v ${PWD}/.ci/load/scripts:/locust locustio/locust -f /locust/locustfile.py -u 10 --headless
 }
 
 function stopLoad() {
-    # Stop the load test
-    docker ps|egrep locust|awk '{print $1}'|xargs docker kill
+    # This happens as soon as the container exits so there is nothing to kill
+    curl -s -X POST -H "Content-Type: application/json" -d \
+    "{\"app_token\": \""$APP_TOKEN"\", \
+    \"session_token\": \""$SESSION_TOKEN"\", \
+    \"service\": \"load_generation\", \
+    \"hostname\": \"test_app\", \
+    \"port\": \"8080\"}" \
+    $ORCH_URL/api/stop 
 }
 
 
 waitForApp
 buildArgs
 startLoad
+stopLoad
diff --git a/.ci/load/scripts/start.sh b/.ci/load/scripts/start.sh
index a5953c982c..849319e7c7 100755
--- a/.ci/load/scripts/start.sh
+++ b/.ci/load/scripts/start.sh
@@ -2,13 +2,9 @@
 
 # set -exuo pipefail 
 
-function registerSession() {
-    curl -s -X POST -H "Content-Type: application/json" -d \
-    "{\"app_token\": \""$APP_TOKEN"\", \
-    \"service\": \"application\", \
-    \"hostname\": \"test_app\", \
-    \"port\": \"999\"}" \
-    $ORCH_URL/api/register | jq -Mr '.session_created.session'
-}
-
-registerSession
\ No newline at end of file
+curl -s -X POST -H "Content-Type: application/json" -d \
+"{\"app_token\": \""$APP_TOKEN"\", \
+\"service\": \"application\", \
+\"hostname\": \"test_app\", \
+\"port\": \"999\"}" \
+$ORCH_URL/api/register | jq -Mr '.session_created.session'

From 6d725d33606c728279396b886a5ab252fc031d34 Mon Sep 17 00:00:00 2001
From: Mike Place <mike.place@elastic.co>
Date: Wed, 28 Oct 2020 13:11:11 +0100
Subject: [PATCH 05/37] Script to generate parameters for the load generation
 pipeline

---
 .ci/load/Jenkinsfile                     |  71 ++++++++-----
 .ci/load/scripts/locustfile.py           |  14 +--
 .ci/load/scripts/param_gen/gen_params.py | 125 +++++++++++++++++++++++
 3 files changed, 175 insertions(+), 35 deletions(-)
 create mode 100755 .ci/load/scripts/param_gen/gen_params.py

diff --git a/.ci/load/Jenkinsfile b/.ci/load/Jenkinsfile
index 16fc945591..11e55b7092 100644
--- a/.ci/load/Jenkinsfile
+++ b/.ci/load/Jenkinsfile
@@ -15,14 +15,13 @@ pipeline {
 
     }
     parameters {
-    // TODO: Allow for SHA instead?
-    string(name: 'agent_version', defaultValue: 'v1.9.0', description: 'Version of agent. Should correspond to tag, e.g. `v1.9.0`.')
-    // TODO: Drop-down of JVM versions instead
-    string(name: 'jvm_version', defaultValue: '9.0.4', description: 'Version of JVM.')
-    string(name: 'concurrent_requests', defaultValue: '100', description: 'The number of concurrent requests to test with.')
-    // 300 min is the overall timeout so 280 is probably fine.
-    string(name: 'duration', defaultValue: '10', description: 'Test duration in minutes. Max: 280')
-    string(name: 'num_of_runs', defaultValue: '1', description: 'Number of test runs to execute.')
+        // The following snippet is auto-generated. To update it, run the script located in .ci/load/scripts/param_gen and copy in the output
+        choice(choices: ['1.18.1', '1.18.0', '1.18.0.RC1', '1.17.0', '1.16.0', '1.15.0', '1.14.0', '1.13.0', '1.12.0', '1.11.0', '1.10.0', '1.9.0', '1.8.0', '1.7.0', '1.6.1', '1.6.0', '1.5.0', '1.4.0', '1.3.0', '1.2.0', '1.1.0', '1.0.1', '1.0.0', '1.0.0.RC1', '0.7.1', '0.7.0', '0.6.2', '0.6.1', '0.6.0', '0.5.1', '0.1.2', '0.1.1'], name: "APM Java Agent version")
+        choice(choices: ['oracle-10+43-linux', 'oracle-10+46-linux', 'oracle-11+11-linux', 'oracle-11+12-linux', 'oracle-11+13-linux', 'oracle-11+14-linux', 'oracle-11+15-linux', 'oracle-11+16-linux', 'oracle-11+17-linux', 'oracle-11+18-linux', 'oracle-11+19-linux', 'oracle-11+20-linux', 'oracle-11+21-linux', 'oracle-11+22-linux', 'oracle-11+23-linux', 'oracle-11+24-linux', 'oracle-11+25-linux', 'oracle-11+26-linux', 'oracle-11+27-linux', 'oracle-11+28-linux', 'oracle-11+5-linux', 'oracle-11.0.2+7-linux', 'oracle-11.0.2+9-linux', 'oracle-11.0.3+12-linux', 'oracle-11.0.4+10-linux', 'oracle-11.0.5+10-linux', 'oracle-11.0.6+8-linux', 'oracle-12+33-linux', 'oracle-12.0.1+12-linux', 'oracle-12.0.2+10-linux', 'oracle-13+33-linux', 'oracle-13.0.1+9-linux', 'oracle-13.0.2+8-linux', 'oracle-7u80-linux', 'oracle-8u161-linux', 'oracle-8u162-linux', 'oracle-8u171-linux', 'oracle-8u172-linux', 'oracle-8u181-linux', 'oracle-8u191-linux', 'oracle-8u192-linux', 'oracle-8u201-linux', 'oracle-8u202-linux', 'oracle-8u211-linux', 'oracle-8u212-linux', 'oracle-8u221-linux', 'oracle-8u231-linux', 'oracle-8u241-linux', 'oracle-9.0.4+11-linux'], name: "JVM")
+        string(name: "concurrent_requests", defaultValue: "100", description: "The number of concurrent requests to test with")
+        string(name: "duration", defaultValue: "10", description: "Test duration in minutes. Max: 280")
+        string(name: "num_of_runs", defaultValue: "1", description: "Number of test runs to execute")
+        // End script auto-generation
     }
 
     stages {
@@ -40,34 +39,58 @@ pipeline {
         }
         stage('Load test') {
             parallel {
-                stage('Start load generation') {
+                stage('Load generation') {
                     agent { label 'linux && immutable' }
                     steps {
                             withSecretVault(secret: 'secret/apm-team/ci/bandstand', user_var_name: 'APP_TOKEN_TYPE', pass_var_name: 'APP_TOKEN'){
+                                echo 'Preparing load generation..'
                                 sh(script: ".ci/load/scripts/load_agent.sh")
                             }
                     }
                 }
-                stage('Start application') {
-                    steps {
-                        gitCheckout(
-                            basedir: "${APP_BASE_DIR}",
-                            branch: 'main',
-                            repo: "https://github.com/spring-projects/${APP}.git",
-                            credentialsId: 'f6c7695a-671e-4f4f-a331-acdce44ff9ba',
-                            shallow: false
-                        )
-                        dir("${APP_BASE_DIR}"){
-                            // Launch app in background
-                            sh(script: "./mvnw spring-boot:run &")
+                stage('Test application') {
+                    stages{
+                        stage('Provision Java') {
+                            steps {
+                                echo 'Provision java'
+                            }
+                        }
+                        stage('Provision test application') {
+                            steps {
+                                echo 'Checking out test application'
+                                gitCheckout(
+                                    basedir: "${APP_BASE_DIR}",
+                                    branch: 'main',
+                                    repo: "https://github.com/spring-projects/${APP}.git",
+                                    credentialsId: 'f6c7695a-671e-4f4f-a331-acdce44ff9ba',
+                                    shallow: false
+                                )
+                            }
                         }
-                        // Foreground the orchestrator script for execution control
-                        withSecretVault(secret: 'secret/apm-team/ci/bandstand', user_var_name: 'APP_TOKEN_TYPE', pass_var_name: 'APP_TOKEN'){
-                            sh(script: ".ci/load/scripts/app.sh")
+                        stage('Application load') {
+                            steps {
+                                echo 'Starting test application in background..'
+                                dir("${APP_BASE_DIR}"){
+                                    // Launch app in background
+                                    sh(script: "./mvnw spring-boot:run &")
+                                }
+                                echo 'Starting bandstand orchestrator..'
+                                // Foreground the orchestrator script for execution control
+                                withSecretVault(secret: 'secret/apm-team/ci/bandstand', user_var_name: 'APP_TOKEN_TYPE', pass_var_name: 'APP_TOKEN'){
+                                    sh(script: ".ci/load/scripts/app.sh")
+                                }
+                            }
                         }
                     }
+
                 }
             }
         }
+        stage('Collect results') {
+            steps{
+                echo "Implement result collection"
+                echo "To view results, JMC is required. Get it here: https://jdk.java.net/jmc/"
+            }
+        }
     }
 }
\ No newline at end of file
diff --git a/.ci/load/scripts/locustfile.py b/.ci/load/scripts/locustfile.py
index 1ba93edfb3..bc28fa9453 100644
--- a/.ci/load/scripts/locustfile.py
+++ b/.ci/load/scripts/locustfile.py
@@ -1,4 +1,3 @@
-import time
 from locust import HttpUser, task, between
 
 
@@ -7,15 +6,8 @@ class QuickstartUser(HttpUser):
 
     @task
     def index_page(self):
-        self.client.get("/hello")
-        self.client.get("/world")
+        self.client.get("/owners/find")
 
     @task(3)
-    def view_item(self):
-        for item_id in range(10):
-            self.client.get(f"/item?id={item_id}", name="/item")
-            time.sleep(1)
-
-    def on_start(self):
-        self.client.post("/login", json={"username":"foo", "password":"bar"})
-
+    def gen_error(self):
+        self.client.get("/oups")
diff --git a/.ci/load/scripts/param_gen/gen_params.py b/.ci/load/scripts/param_gen/gen_params.py
new file mode 100755
index 0000000000..2d91ad3017
--- /dev/null
+++ b/.ci/load/scripts/param_gen/gen_params.py
@@ -0,0 +1,125 @@
+#!/usr/bin/env python
+
+
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Description:
+
+This is an ad-hoc script used to generate the drop-down selections
+for the Jenkinsfile to allow users to test against various combinations
+of JDKs available in the Elastic JVM catalog which is hosted at:
+https://jvm-catalog.elastic.co.
+
+It dymamically generates a list of available JDKs and of APM Java Agent
+releases. These are returns as a snippet of code which can then be pasted
+directly into a Jenkinsfile. Below is an example of the kind of output
+produced by this script:
+
+    parameters {
+    string(name: 'agent_version', defaultValue: 'v1.9.0', description: 'Version of agent. Should correspond to tag, e.g. `v1.9.0`.')
+    string(name: 'jvm_version', defaultValue: '9.0.4', description: 'Version of JVM.')
+    string(name: 'concurrent_requests', defaultValue: '100', description: 'The number of concurrent requests to test with.')
+    string(name: 'duration', defaultValue: '10', description: 'Test duration in minutes. Max: 280')
+    string(name: 'num_of_runs', defaultValue: '1', description: 'Number of test runs to execute.')
+    }
+
+Author: Mike Place <mike.place@elastic.co>
+
+Maintainers: Observability Developer Productivity
+<observability-robots@elastic.co>
+"""
+
+import os
+from github.Repository import Repository
+import requests
+import argparse
+import github
+
+CATALOG_URL = 'https://jvm-catalog.elastic.co'
+SUPPORTED_JDKS = ['oracle']
+
+parser = argparse.ArgumentParser(description="Jenkins JDK snippet generator")
+parser.add_argument(
+    '--platforms',
+    nargs='+',
+    help='platforms help',
+    type=str,
+    default='linux',
+    choices=['linux', 'darwin', 'windows']
+    )
+parser.add_argument(
+    '--gh-token',
+    help='GitHub token to gather supported releases',
+    type=str,
+    required=False
+)
+
+parsed_args = parser.parse_args()
+
+# print(parsed_args.platforms)
+
+# os_buckets = {
+#     'linux': [],
+#     'darwin': [],
+#     'windows': []
+# }
+
+# Gather JDKs we can support
+r = requests.get(CATALOG_URL + '/jdks')
+if r.status_code != 200:
+    raise Exception('Error encountered trying to download JDK manifest')
+
+supported_jdks = []
+
+for jdk in r.json():
+    # print(jdk)
+    try:
+        flavor, ver, dist = jdk.split('-', 3)
+    except ValueError:
+        flavor, ver, dist, arch = jdk.split('-', 4)
+
+    if dist in parsed_args.platforms and flavor in SUPPORTED_JDKS:
+        supported_jdks.append(jdk)
+
+# print(supported_jdks)
+
+# Gather releases of the agent we can support
+agent_releases = []
+if parsed_args.gh_token:
+    gh = github.Github(login_or_token=parsed_args.gh_token)
+    releases = gh.get_repo('elastic/apm-agent-java').get_releases()
+    for release in releases:
+        _, rel_number = release.title.split(' ')
+        agent_releases.append(rel_number)
+
+
+# Begin to print output
+
+print('Paste the following into the Jenkinsfile:\n\n\n\n')
+
+print(
+    '// The following snippet is auto-generated. To update it, run the script located in .ci/load/scripts/param_gen and copy in the output',  # noqa E501
+    'choice(choices: {}, name: "APM Java Agent version")'.format(agent_releases),  # noqa E501
+    'choice(choices: {}, name: "JVM")'.format(supported_jdks),
+    'string(name: "concurrent_requests", defaultValue: "100", description: "The number of concurrent requests to test with")',  # noqa E501
+    'string(name: "duration", defaultValue: "10", description: "Test duration in minutes. Max: 280")',  # noqa E501
+    'string(name: "num_of_runs", defaultValue: "1", description: "Number of test runs to execute")',  # noqa E501
+    '// End script auto-generation',
+    sep="\n"
+)

From a0d25740866a3f6cbaf619ec6c8cbd8af2ff7930 Mon Sep 17 00:00:00 2001
From: Mike Place <mike.place@elastic.co>
Date: Wed, 28 Oct 2020 15:34:30 +0100
Subject: [PATCH 06/37] Add provisioning for agent

---
 .ci/load/Jenkinsfile                     | 27 +++++++++++++++++++++---
 .ci/load/scripts/param_gen/gen_params.py |  4 ++--
 2 files changed, 26 insertions(+), 5 deletions(-)

diff --git a/.ci/load/Jenkinsfile b/.ci/load/Jenkinsfile
index 11e55b7092..c6adf24fbb 100644
--- a/.ci/load/Jenkinsfile
+++ b/.ci/load/Jenkinsfile
@@ -4,6 +4,7 @@ pipeline {
         REPO = 'apm-agent-java'
         APP = 'spring-petclinic'
         APP_BASE_DIR = "src/${APP}"
+        AGENT_BASE_DIR = "agent/"
         ORCH_URL='10.0.2.2:8000' // FIXME: For development only.
     }
     options {
@@ -16,8 +17,8 @@ pipeline {
     }
     parameters {
         // The following snippet is auto-generated. To update it, run the script located in .ci/load/scripts/param_gen and copy in the output
-        choice(choices: ['1.18.1', '1.18.0', '1.18.0.RC1', '1.17.0', '1.16.0', '1.15.0', '1.14.0', '1.13.0', '1.12.0', '1.11.0', '1.10.0', '1.9.0', '1.8.0', '1.7.0', '1.6.1', '1.6.0', '1.5.0', '1.4.0', '1.3.0', '1.2.0', '1.1.0', '1.0.1', '1.0.0', '1.0.0.RC1', '0.7.1', '0.7.0', '0.6.2', '0.6.1', '0.6.0', '0.5.1', '0.1.2', '0.1.1'], name: "APM Java Agent version")
-        choice(choices: ['oracle-10+43-linux', 'oracle-10+46-linux', 'oracle-11+11-linux', 'oracle-11+12-linux', 'oracle-11+13-linux', 'oracle-11+14-linux', 'oracle-11+15-linux', 'oracle-11+16-linux', 'oracle-11+17-linux', 'oracle-11+18-linux', 'oracle-11+19-linux', 'oracle-11+20-linux', 'oracle-11+21-linux', 'oracle-11+22-linux', 'oracle-11+23-linux', 'oracle-11+24-linux', 'oracle-11+25-linux', 'oracle-11+26-linux', 'oracle-11+27-linux', 'oracle-11+28-linux', 'oracle-11+5-linux', 'oracle-11.0.2+7-linux', 'oracle-11.0.2+9-linux', 'oracle-11.0.3+12-linux', 'oracle-11.0.4+10-linux', 'oracle-11.0.5+10-linux', 'oracle-11.0.6+8-linux', 'oracle-12+33-linux', 'oracle-12.0.1+12-linux', 'oracle-12.0.2+10-linux', 'oracle-13+33-linux', 'oracle-13.0.1+9-linux', 'oracle-13.0.2+8-linux', 'oracle-7u80-linux', 'oracle-8u161-linux', 'oracle-8u162-linux', 'oracle-8u171-linux', 'oracle-8u172-linux', 'oracle-8u181-linux', 'oracle-8u191-linux', 'oracle-8u192-linux', 'oracle-8u201-linux', 'oracle-8u202-linux', 'oracle-8u211-linux', 'oracle-8u212-linux', 'oracle-8u221-linux', 'oracle-8u231-linux', 'oracle-8u241-linux', 'oracle-9.0.4+11-linux'], name: "JVM")
+        choice(choices: ['1.18.1', '1.18.0', '1.18.0.RC1', '1.17.0', '1.16.0', '1.15.0', '1.14.0', '1.13.0', '1.12.0', '1.11.0', '1.10.0', '1.9.0', '1.8.0', '1.7.0', '1.6.1', '1.6.0', '1.5.0', '1.4.0', '1.3.0', '1.2.0', '1.1.0', '1.0.1', '1.0.0', '1.0.0.RC1', '0.7.1', '0.7.0', '0.6.2', '0.6.1', '0.6.0', '0.5.1', '0.1.2', '0.1.1'], name: "apm_version", description: "APM Java Agent version")
+        choice(choices: ['oracle-10+43-linux', 'oracle-10+46-linux', 'oracle-11+11-linux', 'oracle-11+12-linux', 'oracle-11+13-linux', 'oracle-11+14-linux', 'oracle-11+15-linux', 'oracle-11+16-linux', 'oracle-11+17-linux', 'oracle-11+18-linux', 'oracle-11+19-linux', 'oracle-11+20-linux', 'oracle-11+21-linux', 'oracle-11+22-linux', 'oracle-11+23-linux', 'oracle-11+24-linux', 'oracle-11+25-linux', 'oracle-11+26-linux', 'oracle-11+27-linux', 'oracle-11+28-linux', 'oracle-11+5-linux', 'oracle-11.0.2+7-linux', 'oracle-11.0.2+9-linux', 'oracle-11.0.3+12-linux', 'oracle-11.0.4+10-linux', 'oracle-11.0.5+10-linux', 'oracle-11.0.6+8-linux', 'oracle-12+33-linux', 'oracle-12.0.1+12-linux', 'oracle-12.0.2+10-linux', 'oracle-13+33-linux', 'oracle-13.0.1+9-linux', 'oracle-13.0.2+8-linux', 'oracle-7u80-linux', 'oracle-8u161-linux', 'oracle-8u162-linux', 'oracle-8u171-linux', 'oracle-8u172-linux', 'oracle-8u181-linux', 'oracle-8u191-linux', 'oracle-8u192-linux', 'oracle-8u201-linux', 'oracle-8u202-linux', 'oracle-8u211-linux', 'oracle-8u212-linux', 'oracle-8u221-linux', 'oracle-8u231-linux', 'oracle-8u241-linux', 'oracle-9.0.4+11-linux'], name: "jvm_version", description: "JVM")
         string(name: "concurrent_requests", defaultValue: "100", description: "The number of concurrent requests to test with")
         string(name: "duration", defaultValue: "10", description: "Test duration in minutes. Max: 280")
         string(name: "num_of_runs", defaultValue: "1", description: "Number of test runs to execute")
@@ -52,7 +53,27 @@ pipeline {
                     stages{
                         stage('Provision Java') {
                             steps {
-                                echo 'Provision java'
+                                echo 'TODO: Provision java'
+                            }
+                        }
+                        stage ('Provision agent') {
+                            steps {
+                                echo 'Checking out master branch'
+                                dir("${AGENT_BASE_DIR}") {
+                                    gitCheckout(
+                                        basedir: "apm-agent-java",
+                                        branch: 'master',
+                                        repo: "https://github.com/elastic/${REPO}.git",
+                                        credentialsId: 'f6c7695a-671e-4f4f-a331-acdce44ff9ba',
+                                        shallow: false
+                                    )
+                                    echo 'Switching to requested version'
+                                    dir("apm-agent-java"){
+                                        sh(script: "git checkout v${params.apm_version}")
+                                        echo 'Building agent'
+                                        sh(script: './mvnw  clean install -DskipTests=true -Dmaven.javadoc.skip=true')
+                                    }
+                                }
                             }
                         }
                         stage('Provision test application') {
diff --git a/.ci/load/scripts/param_gen/gen_params.py b/.ci/load/scripts/param_gen/gen_params.py
index 2d91ad3017..f100deeca6 100755
--- a/.ci/load/scripts/param_gen/gen_params.py
+++ b/.ci/load/scripts/param_gen/gen_params.py
@@ -115,8 +115,8 @@
 
 print(
     '// The following snippet is auto-generated. To update it, run the script located in .ci/load/scripts/param_gen and copy in the output',  # noqa E501
-    'choice(choices: {}, name: "APM Java Agent version")'.format(agent_releases),  # noqa E501
-    'choice(choices: {}, name: "JVM")'.format(supported_jdks),
+    'choice(choices: {}, name: "apm_version", description: "APM Java Agent version")'.format(agent_releases),  # noqa E501
+    'choice(choices: {}, name: "jvm_version", description: "JVM")'.format(supported_jdks),  # noqa E501
     'string(name: "concurrent_requests", defaultValue: "100", description: "The number of concurrent requests to test with")',  # noqa E501
     'string(name: "duration", defaultValue: "10", description: "Test duration in minutes. Max: 280")',  # noqa E501
     'string(name: "num_of_runs", defaultValue: "1", description: "Number of test runs to execute")',  # noqa E501

From 68f3251b429c46e96749d81ad6fda55a05f9b7b4 Mon Sep 17 00:00:00 2001
From: Mike Place <mike.place@elastic.co>
Date: Wed, 28 Oct 2020 15:36:40 +0100
Subject: [PATCH 07/37] Add licenses for load gen scripts

---
 .ci/load/scripts/app.sh                      | 17 +++++++++++++++++
 .ci/load/scripts/load_agent.sh               | 17 +++++++++++++++++
 .ci/load/scripts/start.sh                    | 17 ++++++++++++++++-
 .ci/load/scripts/tests/simulate_app_start.sh | 17 +++++++++++++++++
 .ci/load/scripts/tests/simulate_register.sh  | 17 +++++++++++++++++
 5 files changed, 84 insertions(+), 1 deletion(-)

diff --git a/.ci/load/scripts/app.sh b/.ci/load/scripts/app.sh
index ee64a270c2..503b3abca7 100755
--- a/.ci/load/scripts/app.sh
+++ b/.ci/load/scripts/app.sh
@@ -1,5 +1,22 @@
 #!/usr/bin/env bash
 
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 set -exuo pipefail 
 
 POLL_FREQ=1
diff --git a/.ci/load/scripts/load_agent.sh b/.ci/load/scripts/load_agent.sh
index a661292462..91591a9d46 100755
--- a/.ci/load/scripts/load_agent.sh
+++ b/.ci/load/scripts/load_agent.sh
@@ -1,5 +1,22 @@
 #!/usr/bin/env bash
 
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 set -exuo pipefail 
 
 POLL_FREQ=1
diff --git a/.ci/load/scripts/start.sh b/.ci/load/scripts/start.sh
index 849319e7c7..4b7d704578 100755
--- a/.ci/load/scripts/start.sh
+++ b/.ci/load/scripts/start.sh
@@ -1,6 +1,21 @@
 #!/usr/bin/env bash
 
-# set -exuo pipefail 
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 curl -s -X POST -H "Content-Type: application/json" -d \
 "{\"app_token\": \""$APP_TOKEN"\", \
diff --git a/.ci/load/scripts/tests/simulate_app_start.sh b/.ci/load/scripts/tests/simulate_app_start.sh
index d443bf4810..2da5f3ba3b 100755
--- a/.ci/load/scripts/tests/simulate_app_start.sh
+++ b/.ci/load/scripts/tests/simulate_app_start.sh
@@ -1,5 +1,22 @@
 #!/usr/bin/env bash
 
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 set -exuo pipefail 
 
 POLL_FREQ=1
diff --git a/.ci/load/scripts/tests/simulate_register.sh b/.ci/load/scripts/tests/simulate_register.sh
index d084e8a249..8ec045ab6e 100755
--- a/.ci/load/scripts/tests/simulate_register.sh
+++ b/.ci/load/scripts/tests/simulate_register.sh
@@ -1,5 +1,22 @@
 #!/usr/bin/env bash
 
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 set -exuo pipefail 
 
 function registerSession() {

From 96f624ddc39f6dcf8277135b9542859d27fe2861 Mon Sep 17 00:00:00 2001
From: Mike Place <mike.place@elastic.co>
Date: Wed, 28 Oct 2020 17:54:40 +0100
Subject: [PATCH 08/37] Dynamic Java provisioning for load testing

---
 .ci/load/Jenkinsfile | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.ci/load/Jenkinsfile b/.ci/load/Jenkinsfile
index c6adf24fbb..48efcc63d1 100644
--- a/.ci/load/Jenkinsfile
+++ b/.ci/load/Jenkinsfile
@@ -53,7 +53,11 @@ pipeline {
                     stages{
                         stage('Provision Java') {
                             steps {
-                                echo 'TODO: Provision java'
+                                echo "Provisioning Java version: ${params.jvm_version}"
+                                script {
+                                    env.JAVA_HOME = sh(script: ".ci/load/scripts/fetch_sdk.sh ${params.jvm_version}", returnStdout: true).trim()
+                                    sh(script: "printenv")
+                                }
                             }
                         }
                         stage ('Provision agent') {

From 51ab260a0b19222de673ef6827d7f59be922edb0 Mon Sep 17 00:00:00 2001
From: Mike Place <mike.place@elastic.co>
Date: Wed, 28 Oct 2020 17:54:59 +0100
Subject: [PATCH 09/37] Script for dynamic Java provisioning for load testing

---
 .ci/load/scripts/fetch_sdk.sh | 58 +++++++++++++++++++++++++++++++++++
 1 file changed, 58 insertions(+)
 create mode 100755 .ci/load/scripts/fetch_sdk.sh

diff --git a/.ci/load/scripts/fetch_sdk.sh b/.ci/load/scripts/fetch_sdk.sh
new file mode 100755
index 0000000000..eb58844705
--- /dev/null
+++ b/.ci/load/scripts/fetch_sdk.sh
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# ================================================================
+
+# Call this script with an SDK to grab. The initial list should
+# be populated in the Jenkinsfile by executing the script in the
+# `param_gen` directory.
+
+# This script will download and unpack whatever SDK is requested
+# and then return to the caller the complete name of the directory
+# which should be available to set as JAVA_HOME.
+
+# Example execution for retreiving the SDK:
+
+# ./fetch_sdk.sh oracle-13+33-linux
+#
+# This script requires the following tooling to be available on the
+# system path prior to execution:
+#
+# 1. cURL   [https://curl.haxx.se/] 
+# 2. jq     [https://stedolan.github.io/jq/]
+# 3. tar    [https://www.gnu.org/software/tar/]
+# ================================================================
+
+# set -exuo pipefail 
+
+
+CATALOG_URL="https://jvm-catalog.elastic.co/jdks"
+
+read -r SDK_URL SDK_FILENAME <<<$(curl -s $CATALOG_URL|jq -Mr '.['\"$1\"'].url, .['\"$1\"'].filename')
+
+curl -s -o $SDK_FILENAME $SDK_URL
+
+if [ ! -e tmp_java ]; then
+    mkdir tmp_java/
+fi
+
+tar xfz $SDK_FILENAME -C tmp_java/
+
+UNPACKED_JDK_DIR=$(ls tmp_java)
+echo $PWD/tmp_java/$UNPACKED_JDK_DIR

From 7c6f52f36c242b3fc29dab073c1fcbf9d10700d4 Mon Sep 17 00:00:00 2001
From: Mike Place <mike.place@elastic.co>
Date: Thu, 29 Oct 2020 14:53:21 +0100
Subject: [PATCH 10/37] Fix terrible bug with JDK selection and add more JDKs

---
 .ci/load/Jenkinsfile                     | 28 ++++++++++++++----------
 .ci/load/scripts/param_gen/gen_params.py |  2 +-
 2 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/.ci/load/Jenkinsfile b/.ci/load/Jenkinsfile
index 48efcc63d1..9ca53d11d6 100644
--- a/.ci/load/Jenkinsfile
+++ b/.ci/load/Jenkinsfile
@@ -18,7 +18,7 @@ pipeline {
     parameters {
         // The following snippet is auto-generated. To update it, run the script located in .ci/load/scripts/param_gen and copy in the output
         choice(choices: ['1.18.1', '1.18.0', '1.18.0.RC1', '1.17.0', '1.16.0', '1.15.0', '1.14.0', '1.13.0', '1.12.0', '1.11.0', '1.10.0', '1.9.0', '1.8.0', '1.7.0', '1.6.1', '1.6.0', '1.5.0', '1.4.0', '1.3.0', '1.2.0', '1.1.0', '1.0.1', '1.0.0', '1.0.0.RC1', '0.7.1', '0.7.0', '0.6.2', '0.6.1', '0.6.0', '0.5.1', '0.1.2', '0.1.1'], name: "apm_version", description: "APM Java Agent version")
-        choice(choices: ['oracle-10+43-linux', 'oracle-10+46-linux', 'oracle-11+11-linux', 'oracle-11+12-linux', 'oracle-11+13-linux', 'oracle-11+14-linux', 'oracle-11+15-linux', 'oracle-11+16-linux', 'oracle-11+17-linux', 'oracle-11+18-linux', 'oracle-11+19-linux', 'oracle-11+20-linux', 'oracle-11+21-linux', 'oracle-11+22-linux', 'oracle-11+23-linux', 'oracle-11+24-linux', 'oracle-11+25-linux', 'oracle-11+26-linux', 'oracle-11+27-linux', 'oracle-11+28-linux', 'oracle-11+5-linux', 'oracle-11.0.2+7-linux', 'oracle-11.0.2+9-linux', 'oracle-11.0.3+12-linux', 'oracle-11.0.4+10-linux', 'oracle-11.0.5+10-linux', 'oracle-11.0.6+8-linux', 'oracle-12+33-linux', 'oracle-12.0.1+12-linux', 'oracle-12.0.2+10-linux', 'oracle-13+33-linux', 'oracle-13.0.1+9-linux', 'oracle-13.0.2+8-linux', 'oracle-7u80-linux', 'oracle-8u161-linux', 'oracle-8u162-linux', 'oracle-8u171-linux', 'oracle-8u172-linux', 'oracle-8u181-linux', 'oracle-8u191-linux', 'oracle-8u192-linux', 'oracle-8u201-linux', 'oracle-8u202-linux', 'oracle-8u211-linux', 'oracle-8u212-linux', 'oracle-8u221-linux', 'oracle-8u231-linux', 'oracle-8u241-linux', 'oracle-9.0.4+11-linux'], name: "jvm_version", description: "JVM")
+        choice(choices: ['adoptopenjdk-11+28-linux', 'adoptopenjdk-11.0.1+13-linux', 'adoptopenjdk-11.0.1+13-linux-aarch64', 'adoptopenjdk-11.0.2+7-linux', 'adoptopenjdk-11.0.2+7-linux-aarch64', 'adoptopenjdk-11.0.2+9-linux', 'adoptopenjdk-11.0.2+9-linux-aarch64', 'adoptopenjdk-11.0.3+7-linux', 'adoptopenjdk-11.0.3+7-linux-aarch64', 'adoptopenjdk-11.0.4+11-linux', 'adoptopenjdk-11.0.4+11-linux-aarch64', 'adoptopenjdk-11.0.5+10-linux', 'adoptopenjdk-11.0.6+10-linux', 'adoptopenjdk-11.0.6+10-linux-aarch64', 'adoptopenjdk-11.0.7+10-linux', 'adoptopenjdk-11.0.7+10-linux-aarch64', 'adoptopenjdk-11.0.8+10-linux', 'adoptopenjdk-11.0.8+10-linux-aarch64', 'adoptopenjdk-11.0.9+11-linux', 'adoptopenjdk-11.0.9+11-linux-aarch64', 'adoptopenjdk-12+33-linux', 'adoptopenjdk-12.0.1+12-linux', 'adoptopenjdk-12.0.1+12-linux-aarch64', 'adoptopenjdk-12.0.2+10-linux', 'adoptopenjdk-12.0.2+10-linux-aarch64', 'adoptopenjdk-13.0.1+9-linux', 'adoptopenjdk-13.0.2+8-linux', 'adoptopenjdk-13.0.2+8-linux-aarch64', 'adoptopenjdk-14.0.1+7-linux', 'adoptopenjdk-14.0.1+7-linux-aarch64', 'adoptopenjdk-14.0.2+12-linux', 'adoptopenjdk-14.0.2+12-linux-aarch64', 'adoptopenjdk-15+36-linux', 'adoptopenjdk-15+36-linux-aarch64', 'adoptopenjdk-15.0.1+9-linux', 'adoptopenjdk-15.0.1+9-linux-aarch64', 'adoptopenjdk-8u181-linux', 'adoptopenjdk-8u191-linux-aarch64', 'adoptopenjdk-8u192-linux', 'adoptopenjdk-8u202-linux', 'adoptopenjdk-8u212-linux', 'adoptopenjdk-8u222-linux', 'adoptopenjdk-8u222-linux-aarch64', 'adoptopenjdk-8u232-linux', 'adoptopenjdk-8u232-linux-aarch64', 'adoptopenjdk-8u242-linux', 'adoptopenjdk-8u252-linux', 'adoptopenjdk-8u252-linux-aarch64', 'adoptopenjdk-8u262-linux', 'adoptopenjdk-8u262-linux-aarch64', 'adoptopenjdk-8u265-linux', 'adoptopenjdk-8u265-linux-aarch64', 'adoptopenjdk-8u272-linux', 'adoptopenjdk-8u272-linux-aarch64', 'jdk-8u101-linux-x64', 'jdk-8u102-linux-x64', 'jdk-8u121-linux-x64', 'jdk-8u131-linux-x64', 'jdk-8u141-linux-x64', 'jdk-8u144-linux-x64', 'jdk-8u151-linux-x64', 'jdk-8u152-linux-x64', 'jdk-8u161-linux-x64', 'jdk-8u162-linux-x64', 'jdk-8u20-linux-x64', 'jdk-8u45-linux-x64', 'openjdk-10+43-linux', 'openjdk-10-linux', 'openjdk-10.0.1-linux', 'openjdk-10.0.2-linux', 'openjdk-11+11-linux', 'openjdk-11+12-linux', 'openjdk-11+13-linux', 'openjdk-11+14-linux', 'openjdk-11+15-linux', 'openjdk-11+16-linux', 'openjdk-11+17-linux', 'openjdk-11+18-linux', 'openjdk-11+19-linux', 'openjdk-11+20-linux', 'openjdk-11+21-linux', 'openjdk-11+22-linux', 'openjdk-11+23-linux', 'openjdk-11+24-linux', 'openjdk-11+25-linux', 'openjdk-11+26-linux', 'openjdk-11+27-linux', 'openjdk-11+28-linux', 'openjdk-11+5-linux', 'openjdk-11-linux', 'openjdk-11.0.1-linux', 'openjdk-11.0.2-linux', 'openjdk-12+23-linux', 'openjdk-12+24-linux', 'openjdk-12+25-linux', 'openjdk-12+27-linux', 'openjdk-12+28-linux', 'openjdk-12+29-linux', 'openjdk-12+30-linux', 'openjdk-12+31-linux', 'openjdk-12+32-linux', 'openjdk-12+33-linux', 'openjdk-12-linux', 'openjdk-12.0.1-linux', 'openjdk-12.0.2-linux', 'openjdk-13+14-linux', 'openjdk-13+15-linux', 'openjdk-13+16-linux', 'openjdk-13+17-linux', 'openjdk-13+18-linux', 'openjdk-13+19-linux', 'openjdk-13+20-linux', 'openjdk-13+21-linux', 'openjdk-13+22-linux', 'openjdk-13+23-linux', 'openjdk-13+24-linux', 'openjdk-13+25-linux', 'openjdk-13+26-linux', 'openjdk-13+27-linux', 'openjdk-13+28-linux', 'openjdk-13+29-linux', 'openjdk-13+30-linux', 'openjdk-13+31-linux', 'openjdk-13+32-linux', 'openjdk-13-linux', 'openjdk-13.0.1-linux', 'openjdk-13.0.2-linux', 'openjdk-14+10-linux', 'openjdk-14+11-linux', 'openjdk-14+12-linux', 'openjdk-14+13-linux', 'openjdk-14+14-linux', 'openjdk-14+15-linux', 'openjdk-14+16-linux', 'openjdk-14+17-linux', 'openjdk-14+25-linux', 'openjdk-14+26-linux', 'openjdk-14+27-linux', 'openjdk-14+28-linux', 'openjdk-14+30-linux', 'openjdk-14+31-linux', 'openjdk-14+32-linux', 'openjdk-14+33-linux', 'openjdk-14+34-linux', 'openjdk-14+9-linux', 'openjdk-14-linux', 'openjdk-14.0.1-linux', 'openjdk-14.0.2+12-linux', 'openjdk-14.0.2-linux', 'openjdk-15+10-linux', 'openjdk-15+11-linux', 'openjdk-15+12-linux', 'openjdk-15+13-linux', 'openjdk-15+14-linux', 'openjdk-15+15-linux', 'openjdk-15+16-linux', 'openjdk-15+17-linux', 'openjdk-15+18-linux', 'openjdk-15+19-linux', 'openjdk-15+20-linux', 'openjdk-15+21-linux', 'openjdk-15+22-linux', 'openjdk-15+23-linux', 'openjdk-15+24-linux', 'openjdk-15+25-linux', 'openjdk-15+26-linux', 'openjdk-15+27-linux', 'openjdk-15+28-linux', 'openjdk-15+29-linux', 'openjdk-15+30-linux', 'openjdk-15+31-linux', 'openjdk-15+32-linux', 'openjdk-15+33-linux', 'openjdk-15+34-linux', 'openjdk-15+36-linux', 'openjdk-15+4-linux', 'openjdk-15+5-linux', 'openjdk-15+6-linux', 'openjdk-15+7-linux', 'openjdk-15+8-linux', 'openjdk-15+9-linux', 'openjdk-15-linux', 'openjdk-15.0.1+9-linux', 'openjdk-9.0.4-linux', 'oracle-10+43-linux', 'oracle-10+46-linux', 'oracle-11+11-linux', 'oracle-11+12-linux', 'oracle-11+13-linux', 'oracle-11+14-linux', 'oracle-11+15-linux', 'oracle-11+16-linux', 'oracle-11+17-linux', 'oracle-11+18-linux', 'oracle-11+19-linux', 'oracle-11+20-linux', 'oracle-11+21-linux', 'oracle-11+22-linux', 'oracle-11+23-linux', 'oracle-11+24-linux', 'oracle-11+25-linux', 'oracle-11+26-linux', 'oracle-11+27-linux', 'oracle-11+28-linux', 'oracle-11+5-linux', 'oracle-11.0.2+7-linux', 'oracle-11.0.2+9-linux', 'oracle-11.0.3+12-linux', 'oracle-11.0.4+10-linux', 'oracle-11.0.5+10-linux', 'oracle-11.0.6+8-linux', 'oracle-12+33-linux', 'oracle-12.0.1+12-linux', 'oracle-12.0.2+10-linux', 'oracle-13+33-linux', 'oracle-13.0.1+9-linux', 'oracle-13.0.2+8-linux', 'oracle-7u80-linux', 'oracle-8u161-linux', 'oracle-8u162-linux', 'oracle-8u171-linux', 'oracle-8u172-linux', 'oracle-8u181-linux', 'oracle-8u191-linux', 'oracle-8u192-linux', 'oracle-8u201-linux', 'oracle-8u202-linux', 'oracle-8u211-linux', 'oracle-8u212-linux', 'oracle-8u221-linux', 'oracle-8u231-linux', 'oracle-8u241-linux', 'oracle-9.0.4+11-linux', 'zulu-10.0.0-linux', 'zulu-10.0.1-linux', 'zulu-10.0.2-linux', 'zulu-11.0.1-linux', 'zulu-11.0.2-linux', 'zulu-11.0.3-linux', 'zulu-11.0.4-linux', 'zulu-11.0.5-linux', 'zulu-11.0.6-linux', 'zulu-11.0.7-linux', 'zulu-11.0.8-linux', 'zulu-11.0.9-linux', 'zulu-12-linux', 'zulu-12.0.0-linux', 'zulu-12.0.1-linux', 'zulu-12.0.2-linux', 'zulu-13-linux', 'zulu-13.0.0-linux', 'zulu-13.0.1-linux', 'zulu-13.0.2-linux', 'zulu-13.0.3-linux', 'zulu-13.0.4-linux', 'zulu-13.0.5-linux', 'zulu-14-linux', 'zulu-14.0.0-linux', 'zulu-14.0.1-linux', 'zulu-14.0.2-linux', 'zulu-15.0.0-linux', 'zulu-15.0.1-linux', 'zulu-6.0.103-linux', 'zulu-6.0.107-linux', 'zulu-6.0.113-linux', 'zulu-6.0.119-linux', 'zulu-6.0.77-linux', 'zulu-6.0.79-linux', 'zulu-6.0.83-linux', 'zulu-6.0.87-linux', 'zulu-6.0.89-linux', 'zulu-6.0.93-linux', 'zulu-6.0.97-linux', 'zulu-6.0.99-linux', 'zulu-7.0.101-linux', 'zulu-7.0.111-linux', 'zulu-7.0.121-linux', 'zulu-7.0.131-linux', 'zulu-7.0.141-linux', 'zulu-7.0.154-linux', 'zulu-7.0.161-linux', 'zulu-7.0.171-linux', 'zulu-7.0.181-linux', 'zulu-7.0.191-linux', 'zulu-7.0.201-linux', 'zulu-7.0.211-linux', 'zulu-7.0.222-linux', 'zulu-7.0.232-linux', 'zulu-7.0.242-linux', 'zulu-7.0.252-linux', 'zulu-7.0.262-linux', 'zulu-7.0.272-linux', 'zulu-7.0.282-linux', 'zulu-7.0.95-linux', 'zulu-8.0.102-linux', 'zulu-8.0.112-linux', 'zulu-8.0.121-linux', 'zulu-8.0.131-linux', 'zulu-8.0.144-linux', 'zulu-8.0.152-linux', 'zulu-8.0.162-linux', 'zulu-8.0.163-linux', 'zulu-8.0.172-linux', 'zulu-8.0.181-linux', 'zulu-8.0.192-linux', 'zulu-8.0.201-linux', 'zulu-8.0.202-linux', 'zulu-8.0.212-linux', 'zulu-8.0.222-linux', 'zulu-8.0.232-linux', 'zulu-8.0.242-linux', 'zulu-8.0.252-linux', 'zulu-8.0.262-linux', 'zulu-8.0.265-linux', 'zulu-8.0.272-linux', 'zulu-8.0.71-linux', 'zulu-8.0.72-linux', 'zulu-8.0.91-linux', 'zulu-8.0.92-linux', 'zulu-9.0.0-linux', 'zulu-9.0.1-linux', 'zulu-9.0.4-linux', 'zulu-9.0.7-linux'], name: "jvm_version", description: "JVM")
         string(name: "concurrent_requests", defaultValue: "100", description: "The number of concurrent requests to test with")
         string(name: "duration", defaultValue: "10", description: "Test duration in minutes. Max: 280")
         string(name: "num_of_runs", defaultValue: "1", description: "Number of test runs to execute")
@@ -32,6 +32,7 @@ pipeline {
                 withSecretVault(secret: 'secret/apm-team/ci/bandstand', user_var_name: 'APP_TOKEN_TYPE', pass_var_name: 'APP_TOKEN'){
                     script {
                         sh(script: ".ci/load/scripts/start.sh", returnStdout: true)
+                        // TODO pass through to scripts
                         env.LOCUST_RUN_TIME = params.duration
                         env.SESSION_TOKEN = sh(script: ".ci/load/scripts/start.sh", returnStdout: true).trim()
                     }
@@ -55,8 +56,10 @@ pipeline {
                             steps {
                                 echo "Provisioning Java version: ${params.jvm_version}"
                                 script {
-                                    env.JAVA_HOME = sh(script: ".ci/load/scripts/fetch_sdk.sh ${params.jvm_version}", returnStdout: true).trim()
-                                    sh(script: "printenv")
+                                    def jdk_home = sh(script: ".ci/load/scripts/fetch_sdk.sh ${params.jvm_version}", returnStdout: true).trim()
+                                    env.JAVA_HOME = jdk_home
+                                    env.JAVACMD = "${env.JAVA_HOME}/bin/java"
+                                    env.PATH = "${env.JAVA_HOME}/bin:$PATH"
                                 }
                             }
                         }
@@ -97,25 +100,26 @@ pipeline {
                                 echo 'Starting test application in background..'
                                 dir("${APP_BASE_DIR}"){
                                     // Launch app in background
-                                    sh(script: "./mvnw spring-boot:run &")
+                                    withEnv(["MAVEN_OPTS=-XX:+FlightRecorder -XX:StartFlightRecording=filename=flight.jfr"]){
+                                        sh(script: "./mvnw spring-boot:run &")
+                                    }
                                 }
-                                echo 'Starting bandstand orchestrator..'
+                                echo 'Starting bandstand client..'
                                 // Foreground the orchestrator script for execution control
                                 withSecretVault(secret: 'secret/apm-team/ci/bandstand', user_var_name: 'APP_TOKEN_TYPE', pass_var_name: 'APP_TOKEN'){
                                     sh(script: ".ci/load/scripts/app.sh")
                                 }
                             }
                         }
+                        stage('Collecting results') {
+                            steps {
+                                echo "To view results, JMC is required. Get it here: https://jdk.java.net/jmc/"
+                                archiveArtifacts artifacts: "${APP_BASE_DIR}/**/*.jfr"
+                            }
+                        }
                     }
-
                 }
             }
         }
-        stage('Collect results') {
-            steps{
-                echo "Implement result collection"
-                echo "To view results, JMC is required. Get it here: https://jdk.java.net/jmc/"
-            }
-        }
     }
 }
\ No newline at end of file
diff --git a/.ci/load/scripts/param_gen/gen_params.py b/.ci/load/scripts/param_gen/gen_params.py
index f100deeca6..d8b1a82685 100755
--- a/.ci/load/scripts/param_gen/gen_params.py
+++ b/.ci/load/scripts/param_gen/gen_params.py
@@ -52,7 +52,7 @@
 import github
 
 CATALOG_URL = 'https://jvm-catalog.elastic.co'
-SUPPORTED_JDKS = ['oracle']
+SUPPORTED_JDKS = ['oracle', 'openjdk', 'adoptopenjdk', 'amazon', 'jdk', 'zulu']
 
 parser = argparse.ArgumentParser(description="Jenkins JDK snippet generator")
 parser.add_argument(

From 8e7cf645fa2b7bee854abd0b73ac1d93ea6b93db Mon Sep 17 00:00:00 2001
From: Mike Place <mike.place@elastic.co>
Date: Thu, 29 Oct 2020 14:54:20 +0100
Subject: [PATCH 11/37] Remove TODO

---
 .ci/load/Jenkinsfile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.ci/load/Jenkinsfile b/.ci/load/Jenkinsfile
index 9ca53d11d6..e6cb224040 100644
--- a/.ci/load/Jenkinsfile
+++ b/.ci/load/Jenkinsfile
@@ -32,7 +32,6 @@ pipeline {
                 withSecretVault(secret: 'secret/apm-team/ci/bandstand', user_var_name: 'APP_TOKEN_TYPE', pass_var_name: 'APP_TOKEN'){
                     script {
                         sh(script: ".ci/load/scripts/start.sh", returnStdout: true)
-                        // TODO pass through to scripts
                         env.LOCUST_RUN_TIME = params.duration
                         env.SESSION_TOKEN = sh(script: ".ci/load/scripts/start.sh", returnStdout: true).trim()
                     }

From 7f4b66d28b010a4e11a4e3dc31d9a9c1e5efc1a0 Mon Sep 17 00:00:00 2001
From: Mike Place <mike.place@elastic.co>
Date: Thu, 29 Oct 2020 16:25:51 +0100
Subject: [PATCH 12/37] Formatting and lint

---
 .ci/load/Jenkinsfile                         |  2 +-
 .ci/load/scripts/param_gen/gen_params.py     | 16 ----------------
 .ci/load/scripts/tests/simulate_app_start.sh |  2 +-
 .ci/load/scripts/tests/simulate_register.sh  |  2 +-
 4 files changed, 3 insertions(+), 19 deletions(-)

diff --git a/.ci/load/Jenkinsfile b/.ci/load/Jenkinsfile
index e6cb224040..2cd245b10d 100644
--- a/.ci/load/Jenkinsfile
+++ b/.ci/load/Jenkinsfile
@@ -121,4 +121,4 @@ pipeline {
             }
         }
     }
-}
\ No newline at end of file
+}
diff --git a/.ci/load/scripts/param_gen/gen_params.py b/.ci/load/scripts/param_gen/gen_params.py
index d8b1a82685..f6bdceb9ab 100755
--- a/.ci/load/scripts/param_gen/gen_params.py
+++ b/.ci/load/scripts/param_gen/gen_params.py
@@ -1,6 +1,5 @@
 #!/usr/bin/env python
 
-
 # Licensed to Elasticsearch B.V. under one or more contributor
 # license agreements. See the NOTICE file distributed with
 # this work for additional information regarding copyright
@@ -45,8 +44,6 @@
 <observability-robots@elastic.co>
 """
 
-import os
-from github.Repository import Repository
 import requests
 import argparse
 import github
@@ -72,14 +69,6 @@
 
 parsed_args = parser.parse_args()
 
-# print(parsed_args.platforms)
-
-# os_buckets = {
-#     'linux': [],
-#     'darwin': [],
-#     'windows': []
-# }
-
 # Gather JDKs we can support
 r = requests.get(CATALOG_URL + '/jdks')
 if r.status_code != 200:
@@ -97,8 +86,6 @@
     if dist in parsed_args.platforms and flavor in SUPPORTED_JDKS:
         supported_jdks.append(jdk)
 
-# print(supported_jdks)
-
 # Gather releases of the agent we can support
 agent_releases = []
 if parsed_args.gh_token:
@@ -108,9 +95,6 @@
         _, rel_number = release.title.split(' ')
         agent_releases.append(rel_number)
 
-
-# Begin to print output
-
 print('Paste the following into the Jenkinsfile:\n\n\n\n')
 
 print(
diff --git a/.ci/load/scripts/tests/simulate_app_start.sh b/.ci/load/scripts/tests/simulate_app_start.sh
index 2da5f3ba3b..7950473e6e 100755
--- a/.ci/load/scripts/tests/simulate_app_start.sh
+++ b/.ci/load/scripts/tests/simulate_app_start.sh
@@ -31,4 +31,4 @@ function startApp() {
     $ORCH_URL/api/ready 
 }
 
-startApp
\ No newline at end of file
+startApp
diff --git a/.ci/load/scripts/tests/simulate_register.sh b/.ci/load/scripts/tests/simulate_register.sh
index 8ec045ab6e..0ae322f2e8 100755
--- a/.ci/load/scripts/tests/simulate_register.sh
+++ b/.ci/load/scripts/tests/simulate_register.sh
@@ -28,4 +28,4 @@ function registerSession() {
     $ORCH_URL/api/register | jq '.session_created.session'
 }
 
-registerSession
\ No newline at end of file
+registerSession

From 148cb63588b480a97c56cf8a4099c53a1d58ee03 Mon Sep 17 00:00:00 2001
From: Mike Place <mike.place@elastic.co>
Date: Fri, 30 Oct 2020 08:20:40 +0100
Subject: [PATCH 13/37] Add agent_config param

---
 .ci/load/Jenkinsfile                     | 1 +
 .ci/load/scripts/param_gen/gen_params.py | 1 +
 2 files changed, 2 insertions(+)

diff --git a/.ci/load/Jenkinsfile b/.ci/load/Jenkinsfile
index 2cd245b10d..a04d389c71 100644
--- a/.ci/load/Jenkinsfile
+++ b/.ci/load/Jenkinsfile
@@ -22,6 +22,7 @@ pipeline {
         string(name: "concurrent_requests", defaultValue: "100", description: "The number of concurrent requests to test with")
         string(name: "duration", defaultValue: "10", description: "Test duration in minutes. Max: 280")
         string(name: "num_of_runs", defaultValue: "1", description: "Number of test runs to execute")
+        text(name: "agent_config", "defaultValue": "", description: "APM Agent configuration")
         // End script auto-generation
     }
 
diff --git a/.ci/load/scripts/param_gen/gen_params.py b/.ci/load/scripts/param_gen/gen_params.py
index f6bdceb9ab..75e35bac57 100755
--- a/.ci/load/scripts/param_gen/gen_params.py
+++ b/.ci/load/scripts/param_gen/gen_params.py
@@ -104,6 +104,7 @@
     'string(name: "concurrent_requests", defaultValue: "100", description: "The number of concurrent requests to test with")',  # noqa E501
     'string(name: "duration", defaultValue: "10", description: "Test duration in minutes. Max: 280")',  # noqa E501
     'string(name: "num_of_runs", defaultValue: "1", description: "Number of test runs to execute")',  # noqa E501
+    'text(name: "agent_config", "defaultValue": "", description: "APM Agent configuration")',  # noqa E501
     '// End script auto-generation',
     sep="\n"
 )

From 90dd8f6203656ae3d93ce3600c94a5047b63de90 Mon Sep 17 00:00:00 2001
From: Mike Place <mike.place@elastic.co>
Date: Fri, 30 Oct 2020 09:49:33 +0100
Subject: [PATCH 14/37] Add min JDK version and custom config option

---
 .ci/load/Jenkinsfile                     | 15 ++++++++++++---
 .ci/load/scripts/param_gen/gen_params.py | 15 ++++++++++++++-
 2 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/.ci/load/Jenkinsfile b/.ci/load/Jenkinsfile
index a04d389c71..7d2bee8158 100644
--- a/.ci/load/Jenkinsfile
+++ b/.ci/load/Jenkinsfile
@@ -18,11 +18,11 @@ pipeline {
     parameters {
         // The following snippet is auto-generated. To update it, run the script located in .ci/load/scripts/param_gen and copy in the output
         choice(choices: ['1.18.1', '1.18.0', '1.18.0.RC1', '1.17.0', '1.16.0', '1.15.0', '1.14.0', '1.13.0', '1.12.0', '1.11.0', '1.10.0', '1.9.0', '1.8.0', '1.7.0', '1.6.1', '1.6.0', '1.5.0', '1.4.0', '1.3.0', '1.2.0', '1.1.0', '1.0.1', '1.0.0', '1.0.0.RC1', '0.7.1', '0.7.0', '0.6.2', '0.6.1', '0.6.0', '0.5.1', '0.1.2', '0.1.1'], name: "apm_version", description: "APM Java Agent version")
-        choice(choices: ['adoptopenjdk-11+28-linux', 'adoptopenjdk-11.0.1+13-linux', 'adoptopenjdk-11.0.1+13-linux-aarch64', 'adoptopenjdk-11.0.2+7-linux', 'adoptopenjdk-11.0.2+7-linux-aarch64', 'adoptopenjdk-11.0.2+9-linux', 'adoptopenjdk-11.0.2+9-linux-aarch64', 'adoptopenjdk-11.0.3+7-linux', 'adoptopenjdk-11.0.3+7-linux-aarch64', 'adoptopenjdk-11.0.4+11-linux', 'adoptopenjdk-11.0.4+11-linux-aarch64', 'adoptopenjdk-11.0.5+10-linux', 'adoptopenjdk-11.0.6+10-linux', 'adoptopenjdk-11.0.6+10-linux-aarch64', 'adoptopenjdk-11.0.7+10-linux', 'adoptopenjdk-11.0.7+10-linux-aarch64', 'adoptopenjdk-11.0.8+10-linux', 'adoptopenjdk-11.0.8+10-linux-aarch64', 'adoptopenjdk-11.0.9+11-linux', 'adoptopenjdk-11.0.9+11-linux-aarch64', 'adoptopenjdk-12+33-linux', 'adoptopenjdk-12.0.1+12-linux', 'adoptopenjdk-12.0.1+12-linux-aarch64', 'adoptopenjdk-12.0.2+10-linux', 'adoptopenjdk-12.0.2+10-linux-aarch64', 'adoptopenjdk-13.0.1+9-linux', 'adoptopenjdk-13.0.2+8-linux', 'adoptopenjdk-13.0.2+8-linux-aarch64', 'adoptopenjdk-14.0.1+7-linux', 'adoptopenjdk-14.0.1+7-linux-aarch64', 'adoptopenjdk-14.0.2+12-linux', 'adoptopenjdk-14.0.2+12-linux-aarch64', 'adoptopenjdk-15+36-linux', 'adoptopenjdk-15+36-linux-aarch64', 'adoptopenjdk-15.0.1+9-linux', 'adoptopenjdk-15.0.1+9-linux-aarch64', 'adoptopenjdk-8u181-linux', 'adoptopenjdk-8u191-linux-aarch64', 'adoptopenjdk-8u192-linux', 'adoptopenjdk-8u202-linux', 'adoptopenjdk-8u212-linux', 'adoptopenjdk-8u222-linux', 'adoptopenjdk-8u222-linux-aarch64', 'adoptopenjdk-8u232-linux', 'adoptopenjdk-8u232-linux-aarch64', 'adoptopenjdk-8u242-linux', 'adoptopenjdk-8u252-linux', 'adoptopenjdk-8u252-linux-aarch64', 'adoptopenjdk-8u262-linux', 'adoptopenjdk-8u262-linux-aarch64', 'adoptopenjdk-8u265-linux', 'adoptopenjdk-8u265-linux-aarch64', 'adoptopenjdk-8u272-linux', 'adoptopenjdk-8u272-linux-aarch64', 'jdk-8u101-linux-x64', 'jdk-8u102-linux-x64', 'jdk-8u121-linux-x64', 'jdk-8u131-linux-x64', 'jdk-8u141-linux-x64', 'jdk-8u144-linux-x64', 'jdk-8u151-linux-x64', 'jdk-8u152-linux-x64', 'jdk-8u161-linux-x64', 'jdk-8u162-linux-x64', 'jdk-8u20-linux-x64', 'jdk-8u45-linux-x64', 'openjdk-10+43-linux', 'openjdk-10-linux', 'openjdk-10.0.1-linux', 'openjdk-10.0.2-linux', 'openjdk-11+11-linux', 'openjdk-11+12-linux', 'openjdk-11+13-linux', 'openjdk-11+14-linux', 'openjdk-11+15-linux', 'openjdk-11+16-linux', 'openjdk-11+17-linux', 'openjdk-11+18-linux', 'openjdk-11+19-linux', 'openjdk-11+20-linux', 'openjdk-11+21-linux', 'openjdk-11+22-linux', 'openjdk-11+23-linux', 'openjdk-11+24-linux', 'openjdk-11+25-linux', 'openjdk-11+26-linux', 'openjdk-11+27-linux', 'openjdk-11+28-linux', 'openjdk-11+5-linux', 'openjdk-11-linux', 'openjdk-11.0.1-linux', 'openjdk-11.0.2-linux', 'openjdk-12+23-linux', 'openjdk-12+24-linux', 'openjdk-12+25-linux', 'openjdk-12+27-linux', 'openjdk-12+28-linux', 'openjdk-12+29-linux', 'openjdk-12+30-linux', 'openjdk-12+31-linux', 'openjdk-12+32-linux', 'openjdk-12+33-linux', 'openjdk-12-linux', 'openjdk-12.0.1-linux', 'openjdk-12.0.2-linux', 'openjdk-13+14-linux', 'openjdk-13+15-linux', 'openjdk-13+16-linux', 'openjdk-13+17-linux', 'openjdk-13+18-linux', 'openjdk-13+19-linux', 'openjdk-13+20-linux', 'openjdk-13+21-linux', 'openjdk-13+22-linux', 'openjdk-13+23-linux', 'openjdk-13+24-linux', 'openjdk-13+25-linux', 'openjdk-13+26-linux', 'openjdk-13+27-linux', 'openjdk-13+28-linux', 'openjdk-13+29-linux', 'openjdk-13+30-linux', 'openjdk-13+31-linux', 'openjdk-13+32-linux', 'openjdk-13-linux', 'openjdk-13.0.1-linux', 'openjdk-13.0.2-linux', 'openjdk-14+10-linux', 'openjdk-14+11-linux', 'openjdk-14+12-linux', 'openjdk-14+13-linux', 'openjdk-14+14-linux', 'openjdk-14+15-linux', 'openjdk-14+16-linux', 'openjdk-14+17-linux', 'openjdk-14+25-linux', 'openjdk-14+26-linux', 'openjdk-14+27-linux', 'openjdk-14+28-linux', 'openjdk-14+30-linux', 'openjdk-14+31-linux', 'openjdk-14+32-linux', 'openjdk-14+33-linux', 'openjdk-14+34-linux', 'openjdk-14+9-linux', 'openjdk-14-linux', 'openjdk-14.0.1-linux', 'openjdk-14.0.2+12-linux', 'openjdk-14.0.2-linux', 'openjdk-15+10-linux', 'openjdk-15+11-linux', 'openjdk-15+12-linux', 'openjdk-15+13-linux', 'openjdk-15+14-linux', 'openjdk-15+15-linux', 'openjdk-15+16-linux', 'openjdk-15+17-linux', 'openjdk-15+18-linux', 'openjdk-15+19-linux', 'openjdk-15+20-linux', 'openjdk-15+21-linux', 'openjdk-15+22-linux', 'openjdk-15+23-linux', 'openjdk-15+24-linux', 'openjdk-15+25-linux', 'openjdk-15+26-linux', 'openjdk-15+27-linux', 'openjdk-15+28-linux', 'openjdk-15+29-linux', 'openjdk-15+30-linux', 'openjdk-15+31-linux', 'openjdk-15+32-linux', 'openjdk-15+33-linux', 'openjdk-15+34-linux', 'openjdk-15+36-linux', 'openjdk-15+4-linux', 'openjdk-15+5-linux', 'openjdk-15+6-linux', 'openjdk-15+7-linux', 'openjdk-15+8-linux', 'openjdk-15+9-linux', 'openjdk-15-linux', 'openjdk-15.0.1+9-linux', 'openjdk-9.0.4-linux', 'oracle-10+43-linux', 'oracle-10+46-linux', 'oracle-11+11-linux', 'oracle-11+12-linux', 'oracle-11+13-linux', 'oracle-11+14-linux', 'oracle-11+15-linux', 'oracle-11+16-linux', 'oracle-11+17-linux', 'oracle-11+18-linux', 'oracle-11+19-linux', 'oracle-11+20-linux', 'oracle-11+21-linux', 'oracle-11+22-linux', 'oracle-11+23-linux', 'oracle-11+24-linux', 'oracle-11+25-linux', 'oracle-11+26-linux', 'oracle-11+27-linux', 'oracle-11+28-linux', 'oracle-11+5-linux', 'oracle-11.0.2+7-linux', 'oracle-11.0.2+9-linux', 'oracle-11.0.3+12-linux', 'oracle-11.0.4+10-linux', 'oracle-11.0.5+10-linux', 'oracle-11.0.6+8-linux', 'oracle-12+33-linux', 'oracle-12.0.1+12-linux', 'oracle-12.0.2+10-linux', 'oracle-13+33-linux', 'oracle-13.0.1+9-linux', 'oracle-13.0.2+8-linux', 'oracle-7u80-linux', 'oracle-8u161-linux', 'oracle-8u162-linux', 'oracle-8u171-linux', 'oracle-8u172-linux', 'oracle-8u181-linux', 'oracle-8u191-linux', 'oracle-8u192-linux', 'oracle-8u201-linux', 'oracle-8u202-linux', 'oracle-8u211-linux', 'oracle-8u212-linux', 'oracle-8u221-linux', 'oracle-8u231-linux', 'oracle-8u241-linux', 'oracle-9.0.4+11-linux', 'zulu-10.0.0-linux', 'zulu-10.0.1-linux', 'zulu-10.0.2-linux', 'zulu-11.0.1-linux', 'zulu-11.0.2-linux', 'zulu-11.0.3-linux', 'zulu-11.0.4-linux', 'zulu-11.0.5-linux', 'zulu-11.0.6-linux', 'zulu-11.0.7-linux', 'zulu-11.0.8-linux', 'zulu-11.0.9-linux', 'zulu-12-linux', 'zulu-12.0.0-linux', 'zulu-12.0.1-linux', 'zulu-12.0.2-linux', 'zulu-13-linux', 'zulu-13.0.0-linux', 'zulu-13.0.1-linux', 'zulu-13.0.2-linux', 'zulu-13.0.3-linux', 'zulu-13.0.4-linux', 'zulu-13.0.5-linux', 'zulu-14-linux', 'zulu-14.0.0-linux', 'zulu-14.0.1-linux', 'zulu-14.0.2-linux', 'zulu-15.0.0-linux', 'zulu-15.0.1-linux', 'zulu-6.0.103-linux', 'zulu-6.0.107-linux', 'zulu-6.0.113-linux', 'zulu-6.0.119-linux', 'zulu-6.0.77-linux', 'zulu-6.0.79-linux', 'zulu-6.0.83-linux', 'zulu-6.0.87-linux', 'zulu-6.0.89-linux', 'zulu-6.0.93-linux', 'zulu-6.0.97-linux', 'zulu-6.0.99-linux', 'zulu-7.0.101-linux', 'zulu-7.0.111-linux', 'zulu-7.0.121-linux', 'zulu-7.0.131-linux', 'zulu-7.0.141-linux', 'zulu-7.0.154-linux', 'zulu-7.0.161-linux', 'zulu-7.0.171-linux', 'zulu-7.0.181-linux', 'zulu-7.0.191-linux', 'zulu-7.0.201-linux', 'zulu-7.0.211-linux', 'zulu-7.0.222-linux', 'zulu-7.0.232-linux', 'zulu-7.0.242-linux', 'zulu-7.0.252-linux', 'zulu-7.0.262-linux', 'zulu-7.0.272-linux', 'zulu-7.0.282-linux', 'zulu-7.0.95-linux', 'zulu-8.0.102-linux', 'zulu-8.0.112-linux', 'zulu-8.0.121-linux', 'zulu-8.0.131-linux', 'zulu-8.0.144-linux', 'zulu-8.0.152-linux', 'zulu-8.0.162-linux', 'zulu-8.0.163-linux', 'zulu-8.0.172-linux', 'zulu-8.0.181-linux', 'zulu-8.0.192-linux', 'zulu-8.0.201-linux', 'zulu-8.0.202-linux', 'zulu-8.0.212-linux', 'zulu-8.0.222-linux', 'zulu-8.0.232-linux', 'zulu-8.0.242-linux', 'zulu-8.0.252-linux', 'zulu-8.0.262-linux', 'zulu-8.0.265-linux', 'zulu-8.0.272-linux', 'zulu-8.0.71-linux', 'zulu-8.0.72-linux', 'zulu-8.0.91-linux', 'zulu-8.0.92-linux', 'zulu-9.0.0-linux', 'zulu-9.0.1-linux', 'zulu-9.0.4-linux', 'zulu-9.0.7-linux'], name: "jvm_version", description: "JVM")
+        choice(choices: ['adoptopenjdk-11+28-linux', 'adoptopenjdk-11.0.1+13-linux', 'adoptopenjdk-11.0.1+13-linux-aarch64', 'adoptopenjdk-11.0.2+7-linux', 'adoptopenjdk-11.0.2+7-linux-aarch64', 'adoptopenjdk-11.0.2+9-linux', 'adoptopenjdk-11.0.2+9-linux-aarch64', 'adoptopenjdk-11.0.3+7-linux', 'adoptopenjdk-11.0.3+7-linux-aarch64', 'adoptopenjdk-11.0.4+11-linux', 'adoptopenjdk-11.0.4+11-linux-aarch64', 'adoptopenjdk-11.0.5+10-linux', 'adoptopenjdk-11.0.6+10-linux', 'adoptopenjdk-11.0.6+10-linux-aarch64', 'adoptopenjdk-11.0.7+10-linux', 'adoptopenjdk-11.0.7+10-linux-aarch64', 'adoptopenjdk-11.0.8+10-linux', 'adoptopenjdk-11.0.8+10-linux-aarch64', 'adoptopenjdk-11.0.9+11-linux', 'adoptopenjdk-11.0.9+11-linux-aarch64', 'adoptopenjdk-12+33-linux', 'adoptopenjdk-12.0.1+12-linux', 'adoptopenjdk-12.0.1+12-linux-aarch64', 'adoptopenjdk-12.0.2+10-linux', 'adoptopenjdk-12.0.2+10-linux-aarch64', 'adoptopenjdk-13.0.1+9-linux', 'adoptopenjdk-13.0.2+8-linux', 'adoptopenjdk-13.0.2+8-linux-aarch64', 'adoptopenjdk-14.0.1+7-linux', 'adoptopenjdk-14.0.1+7-linux-aarch64', 'adoptopenjdk-14.0.2+12-linux', 'adoptopenjdk-14.0.2+12-linux-aarch64', 'adoptopenjdk-15+36-linux', 'adoptopenjdk-15+36-linux-aarch64', 'adoptopenjdk-15.0.1+9-linux', 'adoptopenjdk-15.0.1+9-linux-aarch64', 'openjdk-11+11-linux', 'openjdk-11+12-linux', 'openjdk-11+13-linux', 'openjdk-11+14-linux', 'openjdk-11+15-linux', 'openjdk-11+16-linux', 'openjdk-11+17-linux', 'openjdk-11+18-linux', 'openjdk-11+19-linux', 'openjdk-11+20-linux', 'openjdk-11+21-linux', 'openjdk-11+22-linux', 'openjdk-11+23-linux', 'openjdk-11+24-linux', 'openjdk-11+25-linux', 'openjdk-11+26-linux', 'openjdk-11+27-linux', 'openjdk-11+28-linux', 'openjdk-11+5-linux', 'openjdk-11-linux', 'openjdk-11.0.1-linux', 'openjdk-11.0.2-linux', 'openjdk-12+23-linux', 'openjdk-12+24-linux', 'openjdk-12+25-linux', 'openjdk-12+27-linux', 'openjdk-12+28-linux', 'openjdk-12+29-linux', 'openjdk-12+30-linux', 'openjdk-12+31-linux', 'openjdk-12+32-linux', 'openjdk-12+33-linux', 'openjdk-12-linux', 'openjdk-12.0.1-linux', 'openjdk-12.0.2-linux', 'openjdk-13+14-linux', 'openjdk-13+15-linux', 'openjdk-13+16-linux', 'openjdk-13+17-linux', 'openjdk-13+18-linux', 'openjdk-13+19-linux', 'openjdk-13+20-linux', 'openjdk-13+21-linux', 'openjdk-13+22-linux', 'openjdk-13+23-linux', 'openjdk-13+24-linux', 'openjdk-13+25-linux', 'openjdk-13+26-linux', 'openjdk-13+27-linux', 'openjdk-13+28-linux', 'openjdk-13+29-linux', 'openjdk-13+30-linux', 'openjdk-13+31-linux', 'openjdk-13+32-linux', 'openjdk-13-linux', 'openjdk-13.0.1-linux', 'openjdk-13.0.2-linux', 'openjdk-14+10-linux', 'openjdk-14+11-linux', 'openjdk-14+12-linux', 'openjdk-14+13-linux', 'openjdk-14+14-linux', 'openjdk-14+15-linux', 'openjdk-14+16-linux', 'openjdk-14+17-linux', 'openjdk-14+25-linux', 'openjdk-14+26-linux', 'openjdk-14+27-linux', 'openjdk-14+28-linux', 'openjdk-14+30-linux', 'openjdk-14+31-linux', 'openjdk-14+32-linux', 'openjdk-14+33-linux', 'openjdk-14+34-linux', 'openjdk-14+9-linux', 'openjdk-14-linux', 'openjdk-14.0.1-linux', 'openjdk-14.0.2+12-linux', 'openjdk-14.0.2-linux', 'openjdk-15+10-linux', 'openjdk-15+11-linux', 'openjdk-15+12-linux', 'openjdk-15+13-linux', 'openjdk-15+14-linux', 'openjdk-15+15-linux', 'openjdk-15+16-linux', 'openjdk-15+17-linux', 'openjdk-15+18-linux', 'openjdk-15+19-linux', 'openjdk-15+20-linux', 'openjdk-15+21-linux', 'openjdk-15+22-linux', 'openjdk-15+23-linux', 'openjdk-15+24-linux', 'openjdk-15+25-linux', 'openjdk-15+26-linux', 'openjdk-15+27-linux', 'openjdk-15+28-linux', 'openjdk-15+29-linux', 'openjdk-15+30-linux', 'openjdk-15+31-linux', 'openjdk-15+32-linux', 'openjdk-15+33-linux', 'openjdk-15+34-linux', 'openjdk-15+36-linux', 'openjdk-15+4-linux', 'openjdk-15+5-linux', 'openjdk-15+6-linux', 'openjdk-15+7-linux', 'openjdk-15+8-linux', 'openjdk-15+9-linux', 'openjdk-15-linux', 'openjdk-15.0.1+9-linux', 'oracle-11+11-linux', 'oracle-11+12-linux', 'oracle-11+13-linux', 'oracle-11+14-linux', 'oracle-11+15-linux', 'oracle-11+16-linux', 'oracle-11+17-linux', 'oracle-11+18-linux', 'oracle-11+19-linux', 'oracle-11+20-linux', 'oracle-11+21-linux', 'oracle-11+22-linux', 'oracle-11+23-linux', 'oracle-11+24-linux', 'oracle-11+25-linux', 'oracle-11+26-linux', 'oracle-11+27-linux', 'oracle-11+28-linux', 'oracle-11+5-linux', 'oracle-11.0.2+7-linux', 'oracle-11.0.2+9-linux', 'oracle-11.0.3+12-linux', 'oracle-11.0.4+10-linux', 'oracle-11.0.5+10-linux', 'oracle-11.0.6+8-linux', 'oracle-12+33-linux', 'oracle-12.0.1+12-linux', 'oracle-12.0.2+10-linux', 'oracle-13+33-linux', 'oracle-13.0.1+9-linux', 'oracle-13.0.2+8-linux', 'zulu-11.0.1-linux', 'zulu-11.0.2-linux', 'zulu-11.0.3-linux', 'zulu-11.0.4-linux', 'zulu-11.0.5-linux', 'zulu-11.0.6-linux', 'zulu-11.0.7-linux', 'zulu-11.0.8-linux', 'zulu-11.0.9-linux', 'zulu-12-linux', 'zulu-12.0.0-linux', 'zulu-12.0.1-linux', 'zulu-12.0.2-linux', 'zulu-13-linux', 'zulu-13.0.0-linux', 'zulu-13.0.1-linux', 'zulu-13.0.2-linux', 'zulu-13.0.3-linux', 'zulu-13.0.4-linux', 'zulu-13.0.5-linux', 'zulu-14-linux', 'zulu-14.0.0-linux', 'zulu-14.0.1-linux', 'zulu-14.0.2-linux', 'zulu-15.0.0-linux', 'zulu-15.0.1-linux'], name: "jvm_version", description: "JVM")
         string(name: "concurrent_requests", defaultValue: "100", description: "The number of concurrent requests to test with")
         string(name: "duration", defaultValue: "10", description: "Test duration in minutes. Max: 280")
         string(name: "num_of_runs", defaultValue: "1", description: "Number of test runs to execute")
-        text(name: "agent_config", "defaultValue": "", description: "APM Agent configuration")
+        text(name: "agent_config", "defaultValue": "", description: "Custom APM Agent configuration. (WARNING: May echo to console. Do not supply sensitive data.)")
         // End script auto-generation
     }
 
@@ -51,6 +51,7 @@ pipeline {
                     }
                 }
                 stage('Test application') {
+                    agent { label 'linux && immutable' }
                     stages{
                         stage('Provision Java') {
                             steps {
@@ -74,13 +75,21 @@ pipeline {
                                         credentialsId: 'f6c7695a-671e-4f4f-a331-acdce44ff9ba',
                                         shallow: false
                                     )
-                                    echo 'Switching to requested version'
                                     dir("apm-agent-java"){
+                                        echo 'Switching to requested version'
                                         sh(script: "git checkout v${params.apm_version}")
                                         echo 'Building agent'
                                         sh(script: './mvnw  clean install -DskipTests=true -Dmaven.javadoc.skip=true')
                                     }
                                 }
+                                script {
+                                    if (params.agent_config) {
+                                        echo 'Writing user-supplied configuration'
+                                        dir("${AGENT_BASE_DIR}") {
+                                            sh script: "echo \"${params.agent_config}\">custom_config.cfg"
+                                        }
+                                    }
+                                }
                             }
                         }
                         stage('Provision test application') {
diff --git a/.ci/load/scripts/param_gen/gen_params.py b/.ci/load/scripts/param_gen/gen_params.py
index 75e35bac57..66dda5d85d 100755
--- a/.ci/load/scripts/param_gen/gen_params.py
+++ b/.ci/load/scripts/param_gen/gen_params.py
@@ -47,9 +47,11 @@
 import requests
 import argparse
 import github
+from packaging.version import parse as parse_version
 
 CATALOG_URL = 'https://jvm-catalog.elastic.co'
 SUPPORTED_JDKS = ['oracle', 'openjdk', 'adoptopenjdk', 'amazon', 'jdk', 'zulu']
+MIN_JDK_VERSION = '11'
 
 parser = argparse.ArgumentParser(description="Jenkins JDK snippet generator")
 parser.add_argument(
@@ -66,9 +68,18 @@
     type=str,
     required=False
 )
+parser.add_argument(
+    '--min-ver',
+    help='Minimum version of JDK to include',
+    type=str,
+    required=False,
+    default=MIN_JDK_VERSION
+)
 
 parsed_args = parser.parse_args()
 
+
+
 # Gather JDKs we can support
 r = requests.get(CATALOG_URL + '/jdks')
 if r.status_code != 200:
@@ -83,7 +94,9 @@
     except ValueError:
         flavor, ver, dist, arch = jdk.split('-', 4)
 
-    if dist in parsed_args.platforms and flavor in SUPPORTED_JDKS:
+    if dist in parsed_args.platforms and \
+        flavor in SUPPORTED_JDKS and \
+            parse_version(ver) >= parse_version(parsed_args.min_ver):
         supported_jdks.append(jdk)
 
 # Gather releases of the agent we can support

From 288e566f850cb80a83c8895ab5c001221e4ff36c Mon Sep 17 00:00:00 2001
From: Mike Place <mike.place@elastic.co>
Date: Fri, 30 Oct 2020 09:54:02 +0100
Subject: [PATCH 15/37] Allow concurrent users setting

---
 .ci/load/Jenkinsfile           | 1 +
 .ci/load/scripts/load_agent.sh | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.ci/load/Jenkinsfile b/.ci/load/Jenkinsfile
index 7d2bee8158..e95671f24d 100644
--- a/.ci/load/Jenkinsfile
+++ b/.ci/load/Jenkinsfile
@@ -34,6 +34,7 @@ pipeline {
                     script {
                         sh(script: ".ci/load/scripts/start.sh", returnStdout: true)
                         env.LOCUST_RUN_TIME = params.duration
+                        env.LOCUST_USERS = params.concurrent_requests
                         env.SESSION_TOKEN = sh(script: ".ci/load/scripts/start.sh", returnStdout: true).trim()
                     }
                 }
diff --git a/.ci/load/scripts/load_agent.sh b/.ci/load/scripts/load_agent.sh
index 91591a9d46..9658d062cd 100755
--- a/.ci/load/scripts/load_agent.sh
+++ b/.ci/load/scripts/load_agent.sh
@@ -70,7 +70,7 @@ function startLoad() {
     \"port\": \"8080\"}" \
     $ORCH_URL/api/ready 
     # FIXME put number of concurrent users in
-    docker run -e "LOCUST_HOST=$LOCUST_HOST" -e "LOCUST_RUN_TIME=$LOCUST_RUN_TIME" -p 8089:8089 -v ${PWD}/.ci/load/scripts:/locust locustio/locust -f /locust/locustfile.py -u 10 --headless
+    docker run -e "LOCUST_HOST=$LOCUST_HOST" -e "LOCUST_RUN_TIME=$LOCUST_RUN_TIME" -e "LOCUST_USERS=$LOCUST_USERS" -p 8089:8089 -v ${PWD}/.ci/load/scripts:/locust locustio/locust -f /locust/locustfile.py -u 10 --headless
 }
 
 function stopLoad() {

From 6898351e5951db441aa33ea1f263dc884e55ac42 Mon Sep 17 00:00:00 2001
From: Mike Place <mike.place@elastic.co>
Date: Fri, 30 Oct 2020 10:03:26 +0100
Subject: [PATCH 16/37] Disable num_of_runs in rev1

---
 .ci/load/Jenkinsfile                     | 4 +++-
 .ci/load/scripts/param_gen/gen_params.py | 3 ++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/.ci/load/Jenkinsfile b/.ci/load/Jenkinsfile
index e95671f24d..98999f93a8 100644
--- a/.ci/load/Jenkinsfile
+++ b/.ci/load/Jenkinsfile
@@ -1,3 +1,4 @@
+// For documentation on this pipeline, please see the README.md in this directory
 pipeline {
     agent any
     environment {
@@ -21,7 +22,8 @@ pipeline {
         choice(choices: ['adoptopenjdk-11+28-linux', 'adoptopenjdk-11.0.1+13-linux', 'adoptopenjdk-11.0.1+13-linux-aarch64', 'adoptopenjdk-11.0.2+7-linux', 'adoptopenjdk-11.0.2+7-linux-aarch64', 'adoptopenjdk-11.0.2+9-linux', 'adoptopenjdk-11.0.2+9-linux-aarch64', 'adoptopenjdk-11.0.3+7-linux', 'adoptopenjdk-11.0.3+7-linux-aarch64', 'adoptopenjdk-11.0.4+11-linux', 'adoptopenjdk-11.0.4+11-linux-aarch64', 'adoptopenjdk-11.0.5+10-linux', 'adoptopenjdk-11.0.6+10-linux', 'adoptopenjdk-11.0.6+10-linux-aarch64', 'adoptopenjdk-11.0.7+10-linux', 'adoptopenjdk-11.0.7+10-linux-aarch64', 'adoptopenjdk-11.0.8+10-linux', 'adoptopenjdk-11.0.8+10-linux-aarch64', 'adoptopenjdk-11.0.9+11-linux', 'adoptopenjdk-11.0.9+11-linux-aarch64', 'adoptopenjdk-12+33-linux', 'adoptopenjdk-12.0.1+12-linux', 'adoptopenjdk-12.0.1+12-linux-aarch64', 'adoptopenjdk-12.0.2+10-linux', 'adoptopenjdk-12.0.2+10-linux-aarch64', 'adoptopenjdk-13.0.1+9-linux', 'adoptopenjdk-13.0.2+8-linux', 'adoptopenjdk-13.0.2+8-linux-aarch64', 'adoptopenjdk-14.0.1+7-linux', 'adoptopenjdk-14.0.1+7-linux-aarch64', 'adoptopenjdk-14.0.2+12-linux', 'adoptopenjdk-14.0.2+12-linux-aarch64', 'adoptopenjdk-15+36-linux', 'adoptopenjdk-15+36-linux-aarch64', 'adoptopenjdk-15.0.1+9-linux', 'adoptopenjdk-15.0.1+9-linux-aarch64', 'openjdk-11+11-linux', 'openjdk-11+12-linux', 'openjdk-11+13-linux', 'openjdk-11+14-linux', 'openjdk-11+15-linux', 'openjdk-11+16-linux', 'openjdk-11+17-linux', 'openjdk-11+18-linux', 'openjdk-11+19-linux', 'openjdk-11+20-linux', 'openjdk-11+21-linux', 'openjdk-11+22-linux', 'openjdk-11+23-linux', 'openjdk-11+24-linux', 'openjdk-11+25-linux', 'openjdk-11+26-linux', 'openjdk-11+27-linux', 'openjdk-11+28-linux', 'openjdk-11+5-linux', 'openjdk-11-linux', 'openjdk-11.0.1-linux', 'openjdk-11.0.2-linux', 'openjdk-12+23-linux', 'openjdk-12+24-linux', 'openjdk-12+25-linux', 'openjdk-12+27-linux', 'openjdk-12+28-linux', 'openjdk-12+29-linux', 'openjdk-12+30-linux', 'openjdk-12+31-linux', 'openjdk-12+32-linux', 'openjdk-12+33-linux', 'openjdk-12-linux', 'openjdk-12.0.1-linux', 'openjdk-12.0.2-linux', 'openjdk-13+14-linux', 'openjdk-13+15-linux', 'openjdk-13+16-linux', 'openjdk-13+17-linux', 'openjdk-13+18-linux', 'openjdk-13+19-linux', 'openjdk-13+20-linux', 'openjdk-13+21-linux', 'openjdk-13+22-linux', 'openjdk-13+23-linux', 'openjdk-13+24-linux', 'openjdk-13+25-linux', 'openjdk-13+26-linux', 'openjdk-13+27-linux', 'openjdk-13+28-linux', 'openjdk-13+29-linux', 'openjdk-13+30-linux', 'openjdk-13+31-linux', 'openjdk-13+32-linux', 'openjdk-13-linux', 'openjdk-13.0.1-linux', 'openjdk-13.0.2-linux', 'openjdk-14+10-linux', 'openjdk-14+11-linux', 'openjdk-14+12-linux', 'openjdk-14+13-linux', 'openjdk-14+14-linux', 'openjdk-14+15-linux', 'openjdk-14+16-linux', 'openjdk-14+17-linux', 'openjdk-14+25-linux', 'openjdk-14+26-linux', 'openjdk-14+27-linux', 'openjdk-14+28-linux', 'openjdk-14+30-linux', 'openjdk-14+31-linux', 'openjdk-14+32-linux', 'openjdk-14+33-linux', 'openjdk-14+34-linux', 'openjdk-14+9-linux', 'openjdk-14-linux', 'openjdk-14.0.1-linux', 'openjdk-14.0.2+12-linux', 'openjdk-14.0.2-linux', 'openjdk-15+10-linux', 'openjdk-15+11-linux', 'openjdk-15+12-linux', 'openjdk-15+13-linux', 'openjdk-15+14-linux', 'openjdk-15+15-linux', 'openjdk-15+16-linux', 'openjdk-15+17-linux', 'openjdk-15+18-linux', 'openjdk-15+19-linux', 'openjdk-15+20-linux', 'openjdk-15+21-linux', 'openjdk-15+22-linux', 'openjdk-15+23-linux', 'openjdk-15+24-linux', 'openjdk-15+25-linux', 'openjdk-15+26-linux', 'openjdk-15+27-linux', 'openjdk-15+28-linux', 'openjdk-15+29-linux', 'openjdk-15+30-linux', 'openjdk-15+31-linux', 'openjdk-15+32-linux', 'openjdk-15+33-linux', 'openjdk-15+34-linux', 'openjdk-15+36-linux', 'openjdk-15+4-linux', 'openjdk-15+5-linux', 'openjdk-15+6-linux', 'openjdk-15+7-linux', 'openjdk-15+8-linux', 'openjdk-15+9-linux', 'openjdk-15-linux', 'openjdk-15.0.1+9-linux', 'oracle-11+11-linux', 'oracle-11+12-linux', 'oracle-11+13-linux', 'oracle-11+14-linux', 'oracle-11+15-linux', 'oracle-11+16-linux', 'oracle-11+17-linux', 'oracle-11+18-linux', 'oracle-11+19-linux', 'oracle-11+20-linux', 'oracle-11+21-linux', 'oracle-11+22-linux', 'oracle-11+23-linux', 'oracle-11+24-linux', 'oracle-11+25-linux', 'oracle-11+26-linux', 'oracle-11+27-linux', 'oracle-11+28-linux', 'oracle-11+5-linux', 'oracle-11.0.2+7-linux', 'oracle-11.0.2+9-linux', 'oracle-11.0.3+12-linux', 'oracle-11.0.4+10-linux', 'oracle-11.0.5+10-linux', 'oracle-11.0.6+8-linux', 'oracle-12+33-linux', 'oracle-12.0.1+12-linux', 'oracle-12.0.2+10-linux', 'oracle-13+33-linux', 'oracle-13.0.1+9-linux', 'oracle-13.0.2+8-linux', 'zulu-11.0.1-linux', 'zulu-11.0.2-linux', 'zulu-11.0.3-linux', 'zulu-11.0.4-linux', 'zulu-11.0.5-linux', 'zulu-11.0.6-linux', 'zulu-11.0.7-linux', 'zulu-11.0.8-linux', 'zulu-11.0.9-linux', 'zulu-12-linux', 'zulu-12.0.0-linux', 'zulu-12.0.1-linux', 'zulu-12.0.2-linux', 'zulu-13-linux', 'zulu-13.0.0-linux', 'zulu-13.0.1-linux', 'zulu-13.0.2-linux', 'zulu-13.0.3-linux', 'zulu-13.0.4-linux', 'zulu-13.0.5-linux', 'zulu-14-linux', 'zulu-14.0.0-linux', 'zulu-14.0.1-linux', 'zulu-14.0.2-linux', 'zulu-15.0.0-linux', 'zulu-15.0.1-linux'], name: "jvm_version", description: "JVM")
         string(name: "concurrent_requests", defaultValue: "100", description: "The number of concurrent requests to test with")
         string(name: "duration", defaultValue: "10", description: "Test duration in minutes. Max: 280")
-        string(name: "num_of_runs", defaultValue: "1", description: "Number of test runs to execute")
+        // num_of_runs currently unsupported
+        // string(name: "num_of_runs", defaultValue: "1", description: "Number of test runs to execute")
         text(name: "agent_config", "defaultValue": "", description: "Custom APM Agent configuration. (WARNING: May echo to console. Do not supply sensitive data.)")
         // End script auto-generation
     }
diff --git a/.ci/load/scripts/param_gen/gen_params.py b/.ci/load/scripts/param_gen/gen_params.py
index 66dda5d85d..f8b1ad720b 100755
--- a/.ci/load/scripts/param_gen/gen_params.py
+++ b/.ci/load/scripts/param_gen/gen_params.py
@@ -116,7 +116,8 @@
     'choice(choices: {}, name: "jvm_version", description: "JVM")'.format(supported_jdks),  # noqa E501
     'string(name: "concurrent_requests", defaultValue: "100", description: "The number of concurrent requests to test with")',  # noqa E501
     'string(name: "duration", defaultValue: "10", description: "Test duration in minutes. Max: 280")',  # noqa E501
-    'string(name: "num_of_runs", defaultValue: "1", description: "Number of test runs to execute")',  # noqa E501
+    '// num_of_runs currently unsupported',
+    '// string(name: "num_of_runs", defaultValue: "1", description: "Number of test runs to execute")',  # noqa E501
     'text(name: "agent_config", "defaultValue": "", description: "APM Agent configuration")',  # noqa E501
     '// End script auto-generation',
     sep="\n"

From 049d33a2bc437f4791aaf93ada6b76e370b15cf0 Mon Sep 17 00:00:00 2001
From: Mike Place <mike.place@elastic.co>
Date: Fri, 30 Oct 2020 10:04:58 +0100
Subject: [PATCH 17/37] Use production orchestrator

---
 .ci/load/Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.ci/load/Jenkinsfile b/.ci/load/Jenkinsfile
index 98999f93a8..b6b82c1ec9 100644
--- a/.ci/load/Jenkinsfile
+++ b/.ci/load/Jenkinsfile
@@ -6,7 +6,7 @@ pipeline {
         APP = 'spring-petclinic'
         APP_BASE_DIR = "src/${APP}"
         AGENT_BASE_DIR = "agent/"
-        ORCH_URL='10.0.2.2:8000' // FIXME: For development only.
+        ORCH_URL='obs-load-orch.app.elastic.co:8000'
     }
     options {
         timeout(time: 3, unit: 'HOURS')

From 72a80395175e74684ff9afc102a9ffa03a1d02ec Mon Sep 17 00:00:00 2001
From: Mike Place <mike.place@elastic.co>
Date: Fri, 30 Oct 2020 11:55:49 +0100
Subject: [PATCH 18/37] More documentation and minor changes

---
 .ci/load/Jenkinsfile                     |  9 +++-
 .ci/load/README.md                       | 60 ++++++++++++++++++++++++
 .ci/load/scripts/load_agent.sh           |  2 -
 .ci/load/scripts/param_gen/gen_params.py |  1 +
 4 files changed, 69 insertions(+), 3 deletions(-)
 create mode 100644 .ci/load/README.md

diff --git a/.ci/load/Jenkinsfile b/.ci/load/Jenkinsfile
index b6b82c1ec9..84fb8b452a 100644
--- a/.ci/load/Jenkinsfile
+++ b/.ci/load/Jenkinsfile
@@ -21,10 +21,11 @@ pipeline {
         choice(choices: ['1.18.1', '1.18.0', '1.18.0.RC1', '1.17.0', '1.16.0', '1.15.0', '1.14.0', '1.13.0', '1.12.0', '1.11.0', '1.10.0', '1.9.0', '1.8.0', '1.7.0', '1.6.1', '1.6.0', '1.5.0', '1.4.0', '1.3.0', '1.2.0', '1.1.0', '1.0.1', '1.0.0', '1.0.0.RC1', '0.7.1', '0.7.0', '0.6.2', '0.6.1', '0.6.0', '0.5.1', '0.1.2', '0.1.1'], name: "apm_version", description: "APM Java Agent version")
         choice(choices: ['adoptopenjdk-11+28-linux', 'adoptopenjdk-11.0.1+13-linux', 'adoptopenjdk-11.0.1+13-linux-aarch64', 'adoptopenjdk-11.0.2+7-linux', 'adoptopenjdk-11.0.2+7-linux-aarch64', 'adoptopenjdk-11.0.2+9-linux', 'adoptopenjdk-11.0.2+9-linux-aarch64', 'adoptopenjdk-11.0.3+7-linux', 'adoptopenjdk-11.0.3+7-linux-aarch64', 'adoptopenjdk-11.0.4+11-linux', 'adoptopenjdk-11.0.4+11-linux-aarch64', 'adoptopenjdk-11.0.5+10-linux', 'adoptopenjdk-11.0.6+10-linux', 'adoptopenjdk-11.0.6+10-linux-aarch64', 'adoptopenjdk-11.0.7+10-linux', 'adoptopenjdk-11.0.7+10-linux-aarch64', 'adoptopenjdk-11.0.8+10-linux', 'adoptopenjdk-11.0.8+10-linux-aarch64', 'adoptopenjdk-11.0.9+11-linux', 'adoptopenjdk-11.0.9+11-linux-aarch64', 'adoptopenjdk-12+33-linux', 'adoptopenjdk-12.0.1+12-linux', 'adoptopenjdk-12.0.1+12-linux-aarch64', 'adoptopenjdk-12.0.2+10-linux', 'adoptopenjdk-12.0.2+10-linux-aarch64', 'adoptopenjdk-13.0.1+9-linux', 'adoptopenjdk-13.0.2+8-linux', 'adoptopenjdk-13.0.2+8-linux-aarch64', 'adoptopenjdk-14.0.1+7-linux', 'adoptopenjdk-14.0.1+7-linux-aarch64', 'adoptopenjdk-14.0.2+12-linux', 'adoptopenjdk-14.0.2+12-linux-aarch64', 'adoptopenjdk-15+36-linux', 'adoptopenjdk-15+36-linux-aarch64', 'adoptopenjdk-15.0.1+9-linux', 'adoptopenjdk-15.0.1+9-linux-aarch64', 'openjdk-11+11-linux', 'openjdk-11+12-linux', 'openjdk-11+13-linux', 'openjdk-11+14-linux', 'openjdk-11+15-linux', 'openjdk-11+16-linux', 'openjdk-11+17-linux', 'openjdk-11+18-linux', 'openjdk-11+19-linux', 'openjdk-11+20-linux', 'openjdk-11+21-linux', 'openjdk-11+22-linux', 'openjdk-11+23-linux', 'openjdk-11+24-linux', 'openjdk-11+25-linux', 'openjdk-11+26-linux', 'openjdk-11+27-linux', 'openjdk-11+28-linux', 'openjdk-11+5-linux', 'openjdk-11-linux', 'openjdk-11.0.1-linux', 'openjdk-11.0.2-linux', 'openjdk-12+23-linux', 'openjdk-12+24-linux', 'openjdk-12+25-linux', 'openjdk-12+27-linux', 'openjdk-12+28-linux', 'openjdk-12+29-linux', 'openjdk-12+30-linux', 'openjdk-12+31-linux', 'openjdk-12+32-linux', 'openjdk-12+33-linux', 'openjdk-12-linux', 'openjdk-12.0.1-linux', 'openjdk-12.0.2-linux', 'openjdk-13+14-linux', 'openjdk-13+15-linux', 'openjdk-13+16-linux', 'openjdk-13+17-linux', 'openjdk-13+18-linux', 'openjdk-13+19-linux', 'openjdk-13+20-linux', 'openjdk-13+21-linux', 'openjdk-13+22-linux', 'openjdk-13+23-linux', 'openjdk-13+24-linux', 'openjdk-13+25-linux', 'openjdk-13+26-linux', 'openjdk-13+27-linux', 'openjdk-13+28-linux', 'openjdk-13+29-linux', 'openjdk-13+30-linux', 'openjdk-13+31-linux', 'openjdk-13+32-linux', 'openjdk-13-linux', 'openjdk-13.0.1-linux', 'openjdk-13.0.2-linux', 'openjdk-14+10-linux', 'openjdk-14+11-linux', 'openjdk-14+12-linux', 'openjdk-14+13-linux', 'openjdk-14+14-linux', 'openjdk-14+15-linux', 'openjdk-14+16-linux', 'openjdk-14+17-linux', 'openjdk-14+25-linux', 'openjdk-14+26-linux', 'openjdk-14+27-linux', 'openjdk-14+28-linux', 'openjdk-14+30-linux', 'openjdk-14+31-linux', 'openjdk-14+32-linux', 'openjdk-14+33-linux', 'openjdk-14+34-linux', 'openjdk-14+9-linux', 'openjdk-14-linux', 'openjdk-14.0.1-linux', 'openjdk-14.0.2+12-linux', 'openjdk-14.0.2-linux', 'openjdk-15+10-linux', 'openjdk-15+11-linux', 'openjdk-15+12-linux', 'openjdk-15+13-linux', 'openjdk-15+14-linux', 'openjdk-15+15-linux', 'openjdk-15+16-linux', 'openjdk-15+17-linux', 'openjdk-15+18-linux', 'openjdk-15+19-linux', 'openjdk-15+20-linux', 'openjdk-15+21-linux', 'openjdk-15+22-linux', 'openjdk-15+23-linux', 'openjdk-15+24-linux', 'openjdk-15+25-linux', 'openjdk-15+26-linux', 'openjdk-15+27-linux', 'openjdk-15+28-linux', 'openjdk-15+29-linux', 'openjdk-15+30-linux', 'openjdk-15+31-linux', 'openjdk-15+32-linux', 'openjdk-15+33-linux', 'openjdk-15+34-linux', 'openjdk-15+36-linux', 'openjdk-15+4-linux', 'openjdk-15+5-linux', 'openjdk-15+6-linux', 'openjdk-15+7-linux', 'openjdk-15+8-linux', 'openjdk-15+9-linux', 'openjdk-15-linux', 'openjdk-15.0.1+9-linux', 'oracle-11+11-linux', 'oracle-11+12-linux', 'oracle-11+13-linux', 'oracle-11+14-linux', 'oracle-11+15-linux', 'oracle-11+16-linux', 'oracle-11+17-linux', 'oracle-11+18-linux', 'oracle-11+19-linux', 'oracle-11+20-linux', 'oracle-11+21-linux', 'oracle-11+22-linux', 'oracle-11+23-linux', 'oracle-11+24-linux', 'oracle-11+25-linux', 'oracle-11+26-linux', 'oracle-11+27-linux', 'oracle-11+28-linux', 'oracle-11+5-linux', 'oracle-11.0.2+7-linux', 'oracle-11.0.2+9-linux', 'oracle-11.0.3+12-linux', 'oracle-11.0.4+10-linux', 'oracle-11.0.5+10-linux', 'oracle-11.0.6+8-linux', 'oracle-12+33-linux', 'oracle-12.0.1+12-linux', 'oracle-12.0.2+10-linux', 'oracle-13+33-linux', 'oracle-13.0.1+9-linux', 'oracle-13.0.2+8-linux', 'zulu-11.0.1-linux', 'zulu-11.0.2-linux', 'zulu-11.0.3-linux', 'zulu-11.0.4-linux', 'zulu-11.0.5-linux', 'zulu-11.0.6-linux', 'zulu-11.0.7-linux', 'zulu-11.0.8-linux', 'zulu-11.0.9-linux', 'zulu-12-linux', 'zulu-12.0.0-linux', 'zulu-12.0.1-linux', 'zulu-12.0.2-linux', 'zulu-13-linux', 'zulu-13.0.0-linux', 'zulu-13.0.1-linux', 'zulu-13.0.2-linux', 'zulu-13.0.3-linux', 'zulu-13.0.4-linux', 'zulu-13.0.5-linux', 'zulu-14-linux', 'zulu-14.0.0-linux', 'zulu-14.0.1-linux', 'zulu-14.0.2-linux', 'zulu-15.0.0-linux', 'zulu-15.0.1-linux'], name: "jvm_version", description: "JVM")
         string(name: "concurrent_requests", defaultValue: "100", description: "The number of concurrent requests to test with")
-        string(name: "duration", defaultValue: "10", description: "Test duration in minutes. Max: 280")
+        string(name: "duration", defaultValue: "10m", description: "Test duration in minutes. Max: 280 minutes")
         // num_of_runs currently unsupported
         // string(name: "num_of_runs", defaultValue: "1", description: "Number of test runs to execute")
         text(name: "agent_config", "defaultValue": "", description: "Custom APM Agent configuration. (WARNING: May echo to console. Do not supply sensitive data.)")
+        text(name: "locustfile", "defaultValue": "", description: "Locust load-generator plan")
         // End script auto-generation
     }
 
@@ -49,6 +50,12 @@ pipeline {
                     steps {
                             withSecretVault(secret: 'secret/apm-team/ci/bandstand', user_var_name: 'APP_TOKEN_TYPE', pass_var_name: 'APP_TOKEN'){
                                 echo 'Preparing load generation..'
+                                script {
+                                    if (params.locustfile) {
+                                        echo 'Using user-supplied plan for load-generation with Locust'
+                                        sh script: "echo \"${params.locustfile}\">.ci/load/scripts/locustfile.py"
+                                    }
+                                }
                                 sh(script: ".ci/load/scripts/load_agent.sh")
                             }
                     }
diff --git a/.ci/load/README.md b/.ci/load/README.md
new file mode 100644
index 0000000000..a29dc3c777
--- /dev/null
+++ b/.ci/load/README.md
@@ -0,0 +1,60 @@
+# APM Java Agent Load Geneator
+
+This directory contains configuration files for load-testing the APM Java Agent.
+
+## Load Tests
+
+Load tests are run using two bare-metal workers. One worker is dedicated to the role of load generation, which other worker is dedicated to the role of hosting the application under test, which is instrumented by the APM Java Agent.
+
+Load generation is performed using [Locust](https://locust.io/) and the application which is instrumented by the APM Java Agent and placed under load is [Spring Petclinic](https://projects.spring.io/spring-petclinic/).
+
+## Orchestration
+
+Because of the nature of these tests, they require the use of a dedicated orchestration layer. For this project, that orchestrator is called Bandstand. It is an internal project to Elastic.
+
+Bandstand performs two primary functions.
+
+1. *Service discovery* The pipeline utilizes a load-generation machine and an application server which run in parallel to each other. These machines need to know how to find each other and Bandstand gives them that information.
+2. *Orchestration* We need to have a system which tells various services when to start an stop based on the state of other services. For example, we can't start load-generation until we are assured that the application is up and in a coherent state. By using an independent orchestrator, we can allow each service to report on its own state. This avoid a number of otherwise difficult-to-maintain dependencies between service state through the lifetime of the load generation.
+
+While the orchestration layer is currently relatively lightweight and simple, it is built to be able to easily extended for more sophisticated needs, such as multiple test runs inside a single test exuection.
+
+The current design uses a persistant orchestator. However, it is a future goal to have a dynamic orchestrator which is unique to each test execution and is spun up and torn down alongside the rest of the services.
+
+## Running Load Tests
+
+To run a load test, navigate to the [APM Java Agent CI](https://apm-ci.elastic.co/job/apm-agent-java/) and look for the `Load Test` pipeline. Tests can be executed with the following parameters:
+
+|Parameter|Description|
+|:-------:|:---------|
+|`apm_version`|The version of the APM Java Agent to use. After checking out the agent from source control, this is the git tag that will be used.|
+|`jvm_version`|The version of the JVM which will be used. This will be used as the version of Java both when building the agent and when launching the application.|
+|`concurrent_requests`|The number of concurrent, simulated users which will make requests against the test applicaiton.|
+|`duration`|The duration for the load test. Specify this value using time units: (300s, 20m, 3h, 1h30m, etc.)|
+|`agent_config`|To override the defaults of the APM Java Agent config, paste a configuration file.|
+|`locustfile`|A Locust file controls the specifics of how load will be applied to the test application, such as which URLs are requested and at what rate. If this option is not specified, then `.ci/load/scripts/locustfile.py` will be used. For more information on writing a locustfile, [please see the documentation](https://docs.locust.io/en/stable/writing-a-locustfile.html).|
+
+
+After specifying the options, a test run will begin. One can view the progress of the test run by following along in the Jenkins Blue Ocean interface for the given job after it is launched. Typically, it takes several minutes for the machines to be provisioned before tests can begin.
+
+## Viewing test results
+
+Because the [test application](https://projects.spring.io/spring-petclinic/) is instrumented with [JFR](https://docs.oracle.com/javacomponents/jmc-5-4/jfr-runtime-guide/about.htm#JFRUH170), it is necessary to have a copy of the [JDK Mission Control application](https://www.baeldung.com/java-flight-recorder-monitoring#3-visualize-data) which is distributed with the JDK.
+
+The results of the JFR instrumentation are available after a test run is complete by navigating to Jenkins and viewing the job results. A link to `Artifacts` will bring you to a page which shows the files generated by the test run. Download `src/spring-petclinic/flight.jfr` and then view its contents with JDK MIssion Control to examine the moment-by-moment profile of the application under test.
+
+To view the load generation output, see the console output in Jenkins for the `Load Generation` step.
+
+## Hacking
+
+*Caution*: The following requires access to libraries which are internal to Elastic. Do not attempt to follow these instructions of you are an open-source user
+of the APM Java Agent. Instead, contact the core development team with questions.
+
+To set up a local development environment for executing load tests and for developing the test framework, perform the following steps:
+
+1. Checkout the Bandstand application and build a docker container from it tagged as `bandstand`.
+2. Checkout the [fork of the APM Pipeline library](https://github.com/cachedout/apm-pipeline-library-1/tree/perf) which contains the necessary modifications to provision a load-testing environment locally.
+3. In the `apm-pipeline-library` checkout from step 3, create a soft link from `local/jenkins_home` to the your checkout of the `apm-java-agent` codebase.
+4. Start up Jenkins using `make start` from your copy of the APM Pipeline library. Start up a worker as well, using the modified `Vagrantfile` from step 2.
+   
+To verify this environment, run `docker ps` and ensure you have a copy of `bandstand` running. 
diff --git a/.ci/load/scripts/load_agent.sh b/.ci/load/scripts/load_agent.sh
index 9658d062cd..8a0304f028 100755
--- a/.ci/load/scripts/load_agent.sh
+++ b/.ci/load/scripts/load_agent.sh
@@ -56,7 +56,6 @@ function buildArgs() {
         \"session_token\": \""$SESSION_TOKEN"\"}" \
         $ORCH_URL/api/poll | \
         jq '.services.application.port')"
-    # FIXME temporary value
     export LOCUST_HOST=http://$(echo $LOCUST_HOSTNAME|sed 's/"//g'):$(echo $LOCUST_PORT|sed 's/"//g')
     export LOCUST_RUN_TIME=30s
 }
@@ -69,7 +68,6 @@ function startLoad() {
     \"hostname\": \"test_app\", \
     \"port\": \"8080\"}" \
     $ORCH_URL/api/ready 
-    # FIXME put number of concurrent users in
     docker run -e "LOCUST_HOST=$LOCUST_HOST" -e "LOCUST_RUN_TIME=$LOCUST_RUN_TIME" -e "LOCUST_USERS=$LOCUST_USERS" -p 8089:8089 -v ${PWD}/.ci/load/scripts:/locust locustio/locust -f /locust/locustfile.py -u 10 --headless
 }
 
diff --git a/.ci/load/scripts/param_gen/gen_params.py b/.ci/load/scripts/param_gen/gen_params.py
index f8b1ad720b..cb7870ac9c 100755
--- a/.ci/load/scripts/param_gen/gen_params.py
+++ b/.ci/load/scripts/param_gen/gen_params.py
@@ -119,6 +119,7 @@
     '// num_of_runs currently unsupported',
     '// string(name: "num_of_runs", defaultValue: "1", description: "Number of test runs to execute")',  # noqa E501
     'text(name: "agent_config", "defaultValue": "", description: "APM Agent configuration")',  # noqa E501
+    'text(name: "locustfile", "defaultValue": "", description: "Locust load-generator plan")',  # noqa E501
     '// End script auto-generation',
     sep="\n"
 )

From 64643e6ad8d716ca9519e46df8c378c6cb4b7c91 Mon Sep 17 00:00:00 2001
From: Mike Place <mike.place@elastic.co>
Date: Fri, 30 Oct 2020 14:48:59 +0100
Subject: [PATCH 19/37] Add bare-metal settings

---
 .ci/load/Jenkinsfile           |  2 +-
 .ci/load/README.md             |  2 +-
 .ci/load/scripts/app.sh        | 61 ++++++++++++++++++++++++++++++++++
 .ci/load/scripts/load_agent.sh | 61 ++++++++++++++++++++++++++++++++++
 4 files changed, 124 insertions(+), 2 deletions(-)

diff --git a/.ci/load/Jenkinsfile b/.ci/load/Jenkinsfile
index 84fb8b452a..78e30af484 100644
--- a/.ci/load/Jenkinsfile
+++ b/.ci/load/Jenkinsfile
@@ -6,7 +6,7 @@ pipeline {
         APP = 'spring-petclinic'
         APP_BASE_DIR = "src/${APP}"
         AGENT_BASE_DIR = "agent/"
-        ORCH_URL='obs-load-orch.app.elastic.co:8000'
+        ORCH_URL='localhost:8000'
     }
     options {
         timeout(time: 3, unit: 'HOURS')
diff --git a/.ci/load/README.md b/.ci/load/README.md
index a29dc3c777..4d7dcbf59f 100644
--- a/.ci/load/README.md
+++ b/.ci/load/README.md
@@ -57,4 +57,4 @@ To set up a local development environment for executing load tests and for devel
 3. In the `apm-pipeline-library` checkout from step 3, create a soft link from `local/jenkins_home` to the your checkout of the `apm-java-agent` codebase.
 4. Start up Jenkins using `make start` from your copy of the APM Pipeline library. Start up a worker as well, using the modified `Vagrantfile` from step 2.
    
-To verify this environment, run `docker ps` and ensure you have a copy of `bandstand` running. 
+To verify this environment, run `docker ps` and ensure you have a copy of `bandstand` running. It may be necessary to manipulate certain variables in the pipeline to avoid looking up certain credentials from the production secret store.
\ No newline at end of file
diff --git a/.ci/load/scripts/app.sh b/.ci/load/scripts/app.sh
index 503b3abca7..d9fc21bd7a 100755
--- a/.ci/load/scripts/app.sh
+++ b/.ci/load/scripts/app.sh
@@ -23,6 +23,52 @@ POLL_FREQ=1
 
 APP_PORT=8080
 
+function setUp() {
+    echo "Setting CPU frequency to base frequency"
+
+    CPU_MODEL=$(lscpu | grep "Model name" | awk '{for(i=3;i<=NF;i++){printf "%s ", $i}; printf "\n"}')
+    if [ "${CPU_MODEL}" == "Intel(R) Xeon(R) CPU E3-1246 v3 @ 3.50GHz " ]
+    then
+        # could also use `nproc`
+        CORE_INDEX=7
+        BASE_FREQ="3.5GHz"
+    elif [ "${CPU_MODEL}" == "Intel(R) Core(TM) i7-6700 CPU @ 3.40GHz " ]
+    then
+        CORE_INDEX=7
+        BASE_FREQ="3.4GHz"
+    elif [ "${CPU_MODEL}" == "Intel(R) Core(TM) i7-7700 CPU @ 3.60GHz " ]
+    then
+        CORE_INDEX=7
+        BASE_FREQ="3.6GHz"
+    elif [ "${CPU_MODEL}" == "Intel(R) Core(TM) i7-8665U CPU @ 1.90GHz " ]
+    then
+        CORE_INDEX=7
+        BASE_FREQ="1.9GHz"
+    else
+        >&2 echo "Cannot determine base frequency for CPU model [${CPU_MODEL}]. Please adjust the build script."
+        exit 1
+    fi
+    MIN_FREQ=$(cpufreq-info -l -c 0 | awk '{print $1}')
+    # This is the frequency including Turbo Boost. See also http://ark.intel.com/products/80916/Intel-Xeon-Processor-E3-1246-v3-8M-Cache-3_50-GHz
+    MAX_FREQ=$(cpufreq-info -l -c 0 | awk '{print $2}')
+
+    # set all CPUs to the base frequency
+    for (( cpu=0; cpu<=${CORE_INDEX}; cpu++ ))
+    do
+        sudo -n cpufreq-set -c ${cpu} --min ${BASE_FREQ} --max ${BASE_FREQ}
+    done
+
+    # Build cgroups to isolate microbenchmarks and JVM threads
+    echo "Creating groups for OS and microbenchmarks"
+    # Isolate the OS to the first core
+    sudo -n cset set --set=/os --cpu=0-1
+    sudo -n cset proc --move --fromset=/ --toset=/os
+
+    # Isolate the microbenchmarks to all cores except the first two (first physical core)
+    # On a 4 core CPU with hyper threading, this would be 6 cores (3 physical cores)
+    sudo -n cset set --set=/benchmark --cpu=2-${CORE_INDEX}
+}
+
 function appIsReady() {
     # Poll the app until it is ready
     curl -Is http://localhost:$APP_PORT/| head -1|egrep 200
@@ -102,6 +148,21 @@ function stopApp() {
     ps -ef|egrep spring-boot:run|egrep java|awk '{print $2}'|xargs kill
 }
 
+function tearDown() {
+    echo "Destroying cgroups"
+    sudo -n cset set --destroy /os
+    sudo -n cset set --destroy /benchmark
+
+    echo "Setting normal frequency range"
+    for (( cpu=0; cpu<=${CORE_INDEX}; cpu++ ))
+    do
+        sudo -n cpufreq-set -c ${cpu} --min ${MIN_FREQ} --max ${MAX_FREQ}
+    done
+}
+
+trap "tearDown" EXIT
+
+setUp
 waitForApp
 sendAppReady
 waitForLoad
diff --git a/.ci/load/scripts/load_agent.sh b/.ci/load/scripts/load_agent.sh
index 8a0304f028..655719986b 100755
--- a/.ci/load/scripts/load_agent.sh
+++ b/.ci/load/scripts/load_agent.sh
@@ -24,6 +24,53 @@ POLL_FREQ=1
 LOCUST_LOCUSTFILE="../locust.py"
 LOCUST_PRINT_STATS=1
 
+
+function setUp() {
+    echo "Setting CPU frequency to base frequency"
+
+    CPU_MODEL=$(lscpu | grep "Model name" | awk '{for(i=3;i<=NF;i++){printf "%s ", $i}; printf "\n"}')
+    if [ "${CPU_MODEL}" == "Intel(R) Xeon(R) CPU E3-1246 v3 @ 3.50GHz " ]
+    then
+        # could also use `nproc`
+        CORE_INDEX=7
+        BASE_FREQ="3.5GHz"
+    elif [ "${CPU_MODEL}" == "Intel(R) Core(TM) i7-6700 CPU @ 3.40GHz " ]
+    then
+        CORE_INDEX=7
+        BASE_FREQ="3.4GHz"
+    elif [ "${CPU_MODEL}" == "Intel(R) Core(TM) i7-7700 CPU @ 3.60GHz " ]
+    then
+        CORE_INDEX=7
+        BASE_FREQ="3.6GHz"
+    elif [ "${CPU_MODEL}" == "Intel(R) Core(TM) i7-8665U CPU @ 1.90GHz " ]
+    then
+        CORE_INDEX=7
+        BASE_FREQ="1.9GHz"
+    else
+        >&2 echo "Cannot determine base frequency for CPU model [${CPU_MODEL}]. Please adjust the build script."
+        exit 1
+    fi
+    MIN_FREQ=$(cpufreq-info -l -c 0 | awk '{print $1}')
+    # This is the frequency including Turbo Boost. See also http://ark.intel.com/products/80916/Intel-Xeon-Processor-E3-1246-v3-8M-Cache-3_50-GHz
+    MAX_FREQ=$(cpufreq-info -l -c 0 | awk '{print $2}')
+
+    # set all CPUs to the base frequency
+    for (( cpu=0; cpu<=${CORE_INDEX}; cpu++ ))
+    do
+        sudo -n cpufreq-set -c ${cpu} --min ${BASE_FREQ} --max ${BASE_FREQ}
+    done
+
+    # Build cgroups to isolate microbenchmarks and JVM threads
+    echo "Creating groups for OS and microbenchmarks"
+    # Isolate the OS to the first core
+    sudo -n cset set --set=/os --cpu=0-1
+    sudo -n cset proc --move --fromset=/ --toset=/os
+
+    # Isolate the microbenchmarks to all cores except the first two (first physical core)
+    # On a 4 core CPU with hyper threading, this would be 6 cores (3 physical cores)
+    sudo -n cset set --set=/benchmark --cpu=2-${CORE_INDEX}
+}
+
 # 1. Loop until we get a signal that the application is ready.
 
 function appIsReady() {
@@ -82,7 +129,21 @@ function stopLoad() {
     $ORCH_URL/api/stop 
 }
 
+function tearDown() {
+    echo "Destroying cgroups"
+    sudo -n cset set --destroy /os
+    sudo -n cset set --destroy /benchmark
+
+    echo "Setting normal frequency range"
+    for (( cpu=0; cpu<=${CORE_INDEX}; cpu++ ))
+    do
+        sudo -n cpufreq-set -c ${cpu} --min ${MIN_FREQ} --max ${MAX_FREQ}
+    done
+}
+
+trap "tearDown" EXIT
 
+setUp
 waitForApp
 buildArgs
 startLoad

From f080229fcad79d4efbfeb42663d4d8053d210e09 Mon Sep 17 00:00:00 2001
From: Mike Place <mike.place@elastic.co>
Date: Mon, 2 Nov 2020 12:26:35 +0100
Subject: [PATCH 20/37] Prep for production

---
 .ci/load/Jenkinsfile                     | 16 ++++++++++++----
 .ci/load/README.md                       |  3 ++-
 .ci/load/scripts/app.sh                  |  7 ++++---
 .ci/load/scripts/load_agent.sh           |  5 +++--
 .ci/load/scripts/param_gen/gen_params.py |  2 +-
 5 files changed, 22 insertions(+), 11 deletions(-)

diff --git a/.ci/load/Jenkinsfile b/.ci/load/Jenkinsfile
index 78e30af484..3f4f800322 100644
--- a/.ci/load/Jenkinsfile
+++ b/.ci/load/Jenkinsfile
@@ -6,7 +6,13 @@ pipeline {
         APP = 'spring-petclinic'
         APP_BASE_DIR = "src/${APP}"
         AGENT_BASE_DIR = "agent/"
-        ORCH_URL='localhost:8000'
+        ORCH_URL = 'obs-load-orch.app.elastic.co:8000'
+        // Set below for local development
+        // ORCH_URL='10.0.2.2:8000'
+        DEBUG_MODE = '0' // set to '0' for production
+        // This is a placeholder server. This will change to a dummy APM Server when it is available.
+        APM_SERVER_URL = 'https://2a2bd0e2806a47e5996eeeec6d22e6df.apm.eu-west-3.aws.elastic-cloud.com:443'
+
     }
     options {
         timeout(time: 3, unit: 'HOURS')
@@ -94,7 +100,7 @@ pipeline {
                                 }
                                 script {
                                     if (params.agent_config) {
-                                        echo 'Writing user-supplied configuration'
+                                        echo 'Writing user-supplied agent configuration'
                                         dir("${AGENT_BASE_DIR}") {
                                             sh script: "echo \"${params.agent_config}\">custom_config.cfg"
                                         }
@@ -119,8 +125,10 @@ pipeline {
                                 echo 'Starting test application in background..'
                                 dir("${APP_BASE_DIR}"){
                                     // Launch app in background
-                                    withEnv(["MAVEN_OPTS=-XX:+FlightRecorder -XX:StartFlightRecording=filename=flight.jfr"]){
-                                        sh(script: "./mvnw spring-boot:run &")
+                                    withSecretVault(secret: 'secret/apm-team/ci/apm-load-test-server', user_var_name: 'APM_TOKEN_TYPE', pass_var_name: 'ELASTIC_APM_API_KEY'){
+                                        // Start with packaging things up
+                                        sh(script: "./mvnw package")
+                                        sh(script: "java -jar -javaagent:${WORKSPACE}/${AGENT_BASE_DIR}/apm-agent-java/elastic-apm-agent/target/elastic-apm-agent-${params.apm_version}.jar -Delastic.apm.server_urls=${env.APM_SERVER_URL} -Delastic.apm.secret_token=${env.ELASTIC_APM_API_KEY} -XX:+FlightRecorder -XX:StartFlightRecording=filename=flight.jfr ./target/spring-petclinic-*.jar &")
                                     }
                                 }
                                 echo 'Starting bandstand client..'
diff --git a/.ci/load/README.md b/.ci/load/README.md
index 4d7dcbf59f..6d61cd7d07 100644
--- a/.ci/load/README.md
+++ b/.ci/load/README.md
@@ -56,5 +56,6 @@ To set up a local development environment for executing load tests and for devel
 2. Checkout the [fork of the APM Pipeline library](https://github.com/cachedout/apm-pipeline-library-1/tree/perf) which contains the necessary modifications to provision a load-testing environment locally.
 3. In the `apm-pipeline-library` checkout from step 3, create a soft link from `local/jenkins_home` to the your checkout of the `apm-java-agent` codebase.
 4. Start up Jenkins using `make start` from your copy of the APM Pipeline library. Start up a worker as well, using the modified `Vagrantfile` from step 2.
+5. Modify the Jenkinsfile to use Bandstand. Set `ORCH_URL` to `10.0.2.2:8000` to use the local instance.
    
-To verify this environment, run `docker ps` and ensure you have a copy of `bandstand` running. It may be necessary to manipulate certain variables in the pipeline to avoid looking up certain credentials from the production secret store.
\ No newline at end of file
+To verify this environment, run `docker ps` and ensure you have a copy of `bandstand` running. It may be necessary to manipulate certain variables in the pipeline to avoid looking up certain credentials from the production secret store.
diff --git a/.ci/load/scripts/app.sh b/.ci/load/scripts/app.sh
index d9fc21bd7a..b5ede85b95 100755
--- a/.ci/load/scripts/app.sh
+++ b/.ci/load/scripts/app.sh
@@ -145,7 +145,7 @@ function checkLoadGenFinish(){
 
 
 function stopApp() {
-    ps -ef|egrep spring-boot:run|egrep java|awk '{print $2}'|xargs kill
+    ps -ef|egrep spring-petclinic|egrep java|awk '{print $2}'|xargs kill
 }
 
 function tearDown() {
@@ -160,9 +160,10 @@ function tearDown() {
     done
 }
 
+if [ ! $DEBUG_MODE ]; then
 trap "tearDown" EXIT
-
-setUp
+    setUp
+fi
 waitForApp
 sendAppReady
 waitForLoad
diff --git a/.ci/load/scripts/load_agent.sh b/.ci/load/scripts/load_agent.sh
index 655719986b..336e6c645f 100755
--- a/.ci/load/scripts/load_agent.sh
+++ b/.ci/load/scripts/load_agent.sh
@@ -140,10 +140,11 @@ function tearDown() {
         sudo -n cpufreq-set -c ${cpu} --min ${MIN_FREQ} --max ${MAX_FREQ}
     done
 }
-
+if [ ! $DEBUG_MODE ]; then
 trap "tearDown" EXIT
-
 setUp
+fi
+
 waitForApp
 buildArgs
 startLoad
diff --git a/.ci/load/scripts/param_gen/gen_params.py b/.ci/load/scripts/param_gen/gen_params.py
index cb7870ac9c..11a755ba98 100755
--- a/.ci/load/scripts/param_gen/gen_params.py
+++ b/.ci/load/scripts/param_gen/gen_params.py
@@ -119,7 +119,7 @@
     '// num_of_runs currently unsupported',
     '// string(name: "num_of_runs", defaultValue: "1", description: "Number of test runs to execute")',  # noqa E501
     'text(name: "agent_config", "defaultValue": "", description: "APM Agent configuration")',  # noqa E501
-    'text(name: "locustfile", "defaultValue": "", description: "Locust load-generator plan")',  # noqa E501
+    'text(name: "locustfile", "defaultValue": "", description: "Locust load-generator plan")',  # noqa E5011
     '// End script auto-generation',
     sep="\n"
 )

From eed86f78167cdb5a2881dac76a882258a26ef4d9 Mon Sep 17 00:00:00 2001
From: Mike Place <mike.place@elastic.co>
Date: Mon, 2 Nov 2020 12:29:09 +0100
Subject: [PATCH 21/37] Add metal tag

---
 .ci/load/Jenkinsfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.ci/load/Jenkinsfile b/.ci/load/Jenkinsfile
index 3f4f800322..b4c58610c5 100644
--- a/.ci/load/Jenkinsfile
+++ b/.ci/load/Jenkinsfile
@@ -50,6 +50,7 @@ pipeline {
             }    
         }
         stage('Load test') {
+            agent { label 'metal' }
             parallel {
                 stage('Load generation') {
                     agent { label 'linux && immutable' }

From 7de00455e17722e56ae5b7bf3bde861fff7572c8 Mon Sep 17 00:00:00 2001
From: Mike Place <mike.place@elastic.co>
Date: Mon, 2 Nov 2020 15:28:28 +0100
Subject: [PATCH 22/37] Add metrics collection

---
 .ci/load/Jenkinsfile                     | 34 +++++++++++++++++++++---
 .ci/load/scripts/param_gen/gen_params.py |  1 +
 2 files changed, 31 insertions(+), 4 deletions(-)

diff --git a/.ci/load/Jenkinsfile b/.ci/load/Jenkinsfile
index b4c58610c5..9e4773f8bf 100644
--- a/.ci/load/Jenkinsfile
+++ b/.ci/load/Jenkinsfile
@@ -5,6 +5,7 @@ pipeline {
         REPO = 'apm-agent-java'
         APP = 'spring-petclinic'
         APP_BASE_DIR = "src/${APP}"
+        METRICS_BASE_DIR="metrics/"
         AGENT_BASE_DIR = "agent/"
         ORCH_URL = 'obs-load-orch.app.elastic.co:8000'
         // Set below for local development
@@ -32,6 +33,7 @@ pipeline {
         // string(name: "num_of_runs", defaultValue: "1", description: "Number of test runs to execute")
         text(name: "agent_config", "defaultValue": "", description: "Custom APM Agent configuration. (WARNING: May echo to console. Do not supply sensitive data.)")
         text(name: "locustfile", "defaultValue": "", description: "Locust load-generator plan")
+        booleanParam(name: "local_metrics", description: "Enable local metrics collection?", defaultValue: false)
         // End script auto-generation
     }
 
@@ -50,10 +52,9 @@ pipeline {
             }    
         }
         stage('Load test') {
-            agent { label 'metal' }
             parallel {
                 stage('Load generation') {
-                    agent { label 'linux && immutable' }
+                    agent { label 'metal' }
                     steps {
                             withSecretVault(secret: 'secret/apm-team/ci/bandstand', user_var_name: 'APP_TOKEN_TYPE', pass_var_name: 'APP_TOKEN'){
                                 echo 'Preparing load generation..'
@@ -68,7 +69,7 @@ pipeline {
                     }
                 }
                 stage('Test application') {
-                    agent { label 'linux && immutable' }
+                    agent { label 'metal' }
                     stages{
                         stage('Provision Java') {
                             steps {
@@ -121,6 +122,29 @@ pipeline {
                                 )
                             }
                         }
+                        stage('Provision local metrics collection') {
+                            when {
+                                expression { 
+                                    return params.local_metrics
+                                }
+                            }
+                            steps {
+                                echo 'Enable local metric collection'
+                                gitCheckout(
+                                    basedir: "${METRICS_BASE_DIR}",
+                                    branch: 'master',
+                                    repo: "https://github.com/pstadler/metrics.sh",
+                                    credentialsId: 'f6c7695a-671e-4f4f-a331-acdce44ff9ba',
+                                    shallow: false
+                                )
+                                sh(script: "touch metrics.out")
+                                dir("${METRICS_BASE_DIR}"){
+                                    withEnv(["FILE_LOCATION=./metrics.out"]) {
+                                        sh(script: "./metrics.sh -r file &")
+                                    }
+                                }
+                            }
+                        }
                         stage('Application load') {
                             steps {
                                 echo 'Starting test application in background..'
@@ -142,7 +166,9 @@ pipeline {
                         stage('Collecting results') {
                             steps {
                                 echo "To view results, JMC is required. Get it here: https://jdk.java.net/jmc/"
-                                archiveArtifacts artifacts: "${APP_BASE_DIR}/**/*.jfr"
+                                archiveArtifacts(allowEmptyArchive: true,
+                                    artifacts: "${APP_BASE_DIR}/**/*.jfr,${METRICS_BASE_DIR}/**/*.out",
+                                    onlyIfSuccessful: false)
                             }
                         }
                     }
diff --git a/.ci/load/scripts/param_gen/gen_params.py b/.ci/load/scripts/param_gen/gen_params.py
index 11a755ba98..c989a8044f 100755
--- a/.ci/load/scripts/param_gen/gen_params.py
+++ b/.ci/load/scripts/param_gen/gen_params.py
@@ -120,6 +120,7 @@
     '// string(name: "num_of_runs", defaultValue: "1", description: "Number of test runs to execute")',  # noqa E501
     'text(name: "agent_config", "defaultValue": "", description: "APM Agent configuration")',  # noqa E501
     'text(name: "locustfile", "defaultValue": "", description: "Locust load-generator plan")',  # noqa E5011
+    'booleanParam(name: "local_metrics", description: "Enable local metrics collection?", defaultValue: false)',  # noqa E501
     '// End script auto-generation',
     sep="\n"
 )

From 4d9aa1bf286bb15613ed1d6f3c7eb4d0a46cf265 Mon Sep 17 00:00:00 2001
From: cachedout <mike.place@elastic.co>
Date: Mon, 2 Nov 2020 16:28:46 +0100
Subject: [PATCH 23/37] Update .ci/load/README.md
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Manuel de la Peña <social.mdelapenya@gmail.com>
---
 .ci/load/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.ci/load/README.md b/.ci/load/README.md
index 6d61cd7d07..9115e3f394 100644
--- a/.ci/load/README.md
+++ b/.ci/load/README.md
@@ -17,7 +17,7 @@ Bandstand performs two primary functions.
 1. *Service discovery* The pipeline utilizes a load-generation machine and an application server which run in parallel to each other. These machines need to know how to find each other and Bandstand gives them that information.
 2. *Orchestration* We need to have a system which tells various services when to start an stop based on the state of other services. For example, we can't start load-generation until we are assured that the application is up and in a coherent state. By using an independent orchestrator, we can allow each service to report on its own state. This avoid a number of otherwise difficult-to-maintain dependencies between service state through the lifetime of the load generation.
 
-While the orchestration layer is currently relatively lightweight and simple, it is built to be able to easily extended for more sophisticated needs, such as multiple test runs inside a single test exuection.
+While the orchestration layer is currently relatively lightweight and simple, it is built to be able to be easily extended for more sophisticated needs, such as multiple test runs inside a single test execution.
 
 The current design uses a persistant orchestator. However, it is a future goal to have a dynamic orchestrator which is unique to each test execution and is spun up and torn down alongside the rest of the services.
 

From 500f89bf7fc09a3bd5246df0db10516e40f917b0 Mon Sep 17 00:00:00 2001
From: cachedout <mike.place@elastic.co>
Date: Mon, 2 Nov 2020 16:28:54 +0100
Subject: [PATCH 24/37] Update .ci/load/README.md
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Manuel de la Peña <social.mdelapenya@gmail.com>
---
 .ci/load/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.ci/load/README.md b/.ci/load/README.md
index 9115e3f394..e16efa537f 100644
--- a/.ci/load/README.md
+++ b/.ci/load/README.md
@@ -54,7 +54,7 @@ To set up a local development environment for executing load tests and for devel
 
 1. Checkout the Bandstand application and build a docker container from it tagged as `bandstand`.
 2. Checkout the [fork of the APM Pipeline library](https://github.com/cachedout/apm-pipeline-library-1/tree/perf) which contains the necessary modifications to provision a load-testing environment locally.
-3. In the `apm-pipeline-library` checkout from step 3, create a soft link from `local/jenkins_home` to the your checkout of the `apm-java-agent` codebase.
+3. In the `apm-pipeline-library` checkout from step 2, create a soft link from `local/jenkins_home` to the your checkout of the `apm-java-agent` codebase.
 4. Start up Jenkins using `make start` from your copy of the APM Pipeline library. Start up a worker as well, using the modified `Vagrantfile` from step 2.
 5. Modify the Jenkinsfile to use Bandstand. Set `ORCH_URL` to `10.0.2.2:8000` to use the local instance.
    

From 61f9670e850341146dde2dd92350623caf2527da Mon Sep 17 00:00:00 2001
From: cachedout <mike.place@elastic.co>
Date: Mon, 2 Nov 2020 16:29:01 +0100
Subject: [PATCH 25/37] Update .ci/load/README.md
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Manuel de la Peña <social.mdelapenya@gmail.com>
---
 .ci/load/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.ci/load/README.md b/.ci/load/README.md
index e16efa537f..61de0b46ad 100644
--- a/.ci/load/README.md
+++ b/.ci/load/README.md
@@ -47,7 +47,7 @@ To view the load generation output, see the console output in Jenkins for the `L
 
 ## Hacking
 
-*Caution*: The following requires access to libraries which are internal to Elastic. Do not attempt to follow these instructions of you are an open-source user
+*Caution*: The following requires access to libraries which are internal to Elastic. Do not attempt to follow these instructions if you are an open-source user
 of the APM Java Agent. Instead, contact the core development team with questions.
 
 To set up a local development environment for executing load tests and for developing the test framework, perform the following steps:

From e51c8e4bd19a3470cc3c8e32317dcff58b141bab Mon Sep 17 00:00:00 2001
From: cachedout <mike.place@elastic.co>
Date: Mon, 2 Nov 2020 17:49:32 +0100
Subject: [PATCH 26/37] Update .ci/load/Jenkinsfile

Co-authored-by: Victor Martinez <victormartinezrubio@gmail.com>
---
 .ci/load/Jenkinsfile | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/.ci/load/Jenkinsfile b/.ci/load/Jenkinsfile
index 9e4773f8bf..e7bd414672 100644
--- a/.ci/load/Jenkinsfile
+++ b/.ci/load/Jenkinsfile
@@ -58,11 +58,9 @@ pipeline {
                     steps {
                             withSecretVault(secret: 'secret/apm-team/ci/bandstand', user_var_name: 'APP_TOKEN_TYPE', pass_var_name: 'APP_TOKEN'){
                                 echo 'Preparing load generation..'
-                                script {
-                                    if (params.locustfile) {
-                                        echo 'Using user-supplied plan for load-generation with Locust'
-                                        sh script: "echo \"${params.locustfile}\">.ci/load/scripts/locustfile.py"
-                                    }
+                                whenTrue(params.locustfile) {
+                                    echo 'Using user-supplied plan for load-generation with Locust'
+                                    sh script: "echo \"${params.locustfile}\">.ci/load/scripts/locustfile.py"
                                 }
                                 sh(script: ".ci/load/scripts/load_agent.sh")
                             }

From 9c6bb6f56a083c9a082881bd08fa16a6357c68f3 Mon Sep 17 00:00:00 2001
From: cachedout <mike.place@elastic.co>
Date: Mon, 2 Nov 2020 17:49:42 +0100
Subject: [PATCH 27/37] Update .ci/load/Jenkinsfile

Co-authored-by: Victor Martinez <victormartinezrubio@gmail.com>
---
 .ci/load/Jenkinsfile | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/.ci/load/Jenkinsfile b/.ci/load/Jenkinsfile
index e7bd414672..76b758ac4f 100644
--- a/.ci/load/Jenkinsfile
+++ b/.ci/load/Jenkinsfile
@@ -98,13 +98,11 @@ pipeline {
                                         sh(script: './mvnw  clean install -DskipTests=true -Dmaven.javadoc.skip=true')
                                     }
                                 }
-                                script {
-                                    if (params.agent_config) {
-                                        echo 'Writing user-supplied agent configuration'
-                                        dir("${AGENT_BASE_DIR}") {
-                                            sh script: "echo \"${params.agent_config}\">custom_config.cfg"
-                                        }
-                                    }
+                                whenTrue(params.agent_config) {
+                                    echo 'Writing user-supplied agent configuration'
+                                    dir("${AGENT_BASE_DIR}") {
+                                        sh script: "echo \"${params.agent_config}\">custom_config.cfg"
+                                     }
                                 }
                             }
                         }

From 0939974512b6a3708ca2c9901bdcfe61367cd440 Mon Sep 17 00:00:00 2001
From: cachedout <mike.place@elastic.co>
Date: Mon, 2 Nov 2020 17:50:24 +0100
Subject: [PATCH 28/37] Update .ci/load/Jenkinsfile

Co-authored-by: Victor Martinez <victormartinezrubio@gmail.com>
---
 .ci/load/Jenkinsfile | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/.ci/load/Jenkinsfile b/.ci/load/Jenkinsfile
index 76b758ac4f..0484b99c10 100644
--- a/.ci/load/Jenkinsfile
+++ b/.ci/load/Jenkinsfile
@@ -72,12 +72,9 @@ pipeline {
                         stage('Provision Java') {
                             steps {
                                 echo "Provisioning Java version: ${params.jvm_version}"
-                                script {
-                                    def jdk_home = sh(script: ".ci/load/scripts/fetch_sdk.sh ${params.jvm_version}", returnStdout: true).trim()
-                                    env.JAVA_HOME = jdk_home
-                                    env.JAVACMD = "${env.JAVA_HOME}/bin/java"
-                                    env.PATH = "${env.JAVA_HOME}/bin:$PATH"
-                                }
+                                setEnvVar('JAVA_HOME', sh(script: ".ci/load/scripts/fetch_sdk.sh ${params.jvm_version}", returnStdout: true).trim())
+                                setEnvVar('JAVACMD', "${env.JAVA_HOME}/bin/java")
+                                setEnvVar('PATH', "${env.JAVA_HOME}/bin:$PATH")
                             }
                         }
                         stage ('Provision agent') {

From e7012da2c09ad2a5d423b43191ecb24296694c01 Mon Sep 17 00:00:00 2001
From: cachedout <mike.place@elastic.co>
Date: Mon, 2 Nov 2020 17:50:40 +0100
Subject: [PATCH 29/37] Update .ci/load/Jenkinsfile

Co-authored-by: Victor Martinez <victormartinezrubio@gmail.com>
---
 .ci/load/Jenkinsfile | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/.ci/load/Jenkinsfile b/.ci/load/Jenkinsfile
index 0484b99c10..d2208b60d8 100644
--- a/.ci/load/Jenkinsfile
+++ b/.ci/load/Jenkinsfile
@@ -42,12 +42,8 @@ pipeline {
             steps {
                 echo 'Getting authentication information from Vault'
                 withSecretVault(secret: 'secret/apm-team/ci/bandstand', user_var_name: 'APP_TOKEN_TYPE', pass_var_name: 'APP_TOKEN'){
-                    script {
-                        sh(script: ".ci/load/scripts/start.sh", returnStdout: true)
-                        env.LOCUST_RUN_TIME = params.duration
-                        env.LOCUST_USERS = params.concurrent_requests
-                        env.SESSION_TOKEN = sh(script: ".ci/load/scripts/start.sh", returnStdout: true).trim()
-                    }
+                    sh(script: ".ci/load/scripts/start.sh", returnStdout: true)
+                    setEnvVar('SESSION_TOKEN', sh(script: ".ci/load/scripts/start.sh", returnStdout: true).trim())
                 }
             }    
         }

From 547e001d254842b04053396c454869d08ea9c5e1 Mon Sep 17 00:00:00 2001
From: Mike Place <mike.place@elastic.co>
Date: Tue, 3 Nov 2020 09:47:37 +0100
Subject: [PATCH 30/37] Fixup of review changes

---
 .ci/load/Jenkinsfile | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.ci/load/Jenkinsfile b/.ci/load/Jenkinsfile
index d2208b60d8..e935bece96 100644
--- a/.ci/load/Jenkinsfile
+++ b/.ci/load/Jenkinsfile
@@ -13,6 +13,8 @@ pipeline {
         DEBUG_MODE = '0' // set to '0' for production
         // This is a placeholder server. This will change to a dummy APM Server when it is available.
         APM_SERVER_URL = 'https://2a2bd0e2806a47e5996eeeec6d22e6df.apm.eu-west-3.aws.elastic-cloud.com:443'
+        LOCUST_RUN_TIME = "${params.duration}"
+        LOCUST_USERS = "${params.concurrent_requests}"
 
     }
     options {
@@ -54,7 +56,7 @@ pipeline {
                     steps {
                             withSecretVault(secret: 'secret/apm-team/ci/bandstand', user_var_name: 'APP_TOKEN_TYPE', pass_var_name: 'APP_TOKEN'){
                                 echo 'Preparing load generation..'
-                                whenTrue(params.locustfile) {
+                                whenTrue(Boolean.valueOf(params.locustfile)) {
                                     echo 'Using user-supplied plan for load-generation with Locust'
                                     sh script: "echo \"${params.locustfile}\">.ci/load/scripts/locustfile.py"
                                 }
@@ -91,7 +93,7 @@ pipeline {
                                         sh(script: './mvnw  clean install -DskipTests=true -Dmaven.javadoc.skip=true')
                                     }
                                 }
-                                whenTrue(params.agent_config) {
+                                whenTrue(Boolean.valueOf(params.agent_config)) {
                                     echo 'Writing user-supplied agent configuration'
                                     dir("${AGENT_BASE_DIR}") {
                                         sh script: "echo \"${params.agent_config}\">custom_config.cfg"

From 090a2596b8d121d2b2bfcb429083ed94a51fc1a2 Mon Sep 17 00:00:00 2001
From: Mike Place <mike.place@elastic.co>
Date: Tue, 3 Nov 2020 10:24:05 +0100
Subject: [PATCH 31/37] Set interval to fixed time

---
 .ci/load/scripts/locustfile.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.ci/load/scripts/locustfile.py b/.ci/load/scripts/locustfile.py
index bc28fa9453..61c3e0e0f8 100644
--- a/.ci/load/scripts/locustfile.py
+++ b/.ci/load/scripts/locustfile.py
@@ -1,8 +1,8 @@
-from locust import HttpUser, task, between
+from locust import HttpUser, task, constant
 
 
 class QuickstartUser(HttpUser):
-    wait_time = between(1, 2)
+    wait_time = constant(1)
 
     @task
     def index_page(self):

From 8329f488a2909b4b4d73dc227981bb7734a4141b Mon Sep 17 00:00:00 2001
From: Mike Place <mike.place@elastic.co>
Date: Tue, 3 Nov 2020 11:39:32 +0100
Subject: [PATCH 32/37] constant pacing for load gen

---
 .ci/load/scripts/locustfile.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.ci/load/scripts/locustfile.py b/.ci/load/scripts/locustfile.py
index 61c3e0e0f8..37c2f39ade 100644
--- a/.ci/load/scripts/locustfile.py
+++ b/.ci/load/scripts/locustfile.py
@@ -1,8 +1,8 @@
-from locust import HttpUser, task, constant
+from locust import HttpUser, task, constant_pacing
 
 
 class QuickstartUser(HttpUser):
-    wait_time = constant(1)
+    wait_time = constant_pacing(1)
 
     @task
     def index_page(self):

From b7d53c2b04c553620901218c40cd5b5904e094cb Mon Sep 17 00:00:00 2001
From: Mike Place <mike.place@elastic.co>
Date: Tue, 3 Nov 2020 11:54:11 +0100
Subject: [PATCH 33/37] Fix bad merge

---
 .ci/load/Jenkinsfile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.ci/load/Jenkinsfile b/.ci/load/Jenkinsfile
index e935bece96..aa868d19b9 100644
--- a/.ci/load/Jenkinsfile
+++ b/.ci/load/Jenkinsfile
@@ -44,7 +44,6 @@ pipeline {
             steps {
                 echo 'Getting authentication information from Vault'
                 withSecretVault(secret: 'secret/apm-team/ci/bandstand', user_var_name: 'APP_TOKEN_TYPE', pass_var_name: 'APP_TOKEN'){
-                    sh(script: ".ci/load/scripts/start.sh", returnStdout: true)
                     setEnvVar('SESSION_TOKEN', sh(script: ".ci/load/scripts/start.sh", returnStdout: true).trim())
                 }
             }    

From cbe6351395b22ec5a3892a4430bfb14901fba624 Mon Sep 17 00:00:00 2001
From: Mike Place <mike.place@elastic.co>
Date: Thu, 5 Nov 2020 14:18:12 +0100
Subject: [PATCH 34/37] Bring number of jdks below 250

---
 .ci/load/Jenkinsfile                     |  4 ++--
 .ci/load/scripts/param_gen/gen_params.py | 17 +++++++++++------
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/.ci/load/Jenkinsfile b/.ci/load/Jenkinsfile
index aa868d19b9..f1d40d8289 100644
--- a/.ci/load/Jenkinsfile
+++ b/.ci/load/Jenkinsfile
@@ -28,9 +28,9 @@ pipeline {
     parameters {
         // The following snippet is auto-generated. To update it, run the script located in .ci/load/scripts/param_gen and copy in the output
         choice(choices: ['1.18.1', '1.18.0', '1.18.0.RC1', '1.17.0', '1.16.0', '1.15.0', '1.14.0', '1.13.0', '1.12.0', '1.11.0', '1.10.0', '1.9.0', '1.8.0', '1.7.0', '1.6.1', '1.6.0', '1.5.0', '1.4.0', '1.3.0', '1.2.0', '1.1.0', '1.0.1', '1.0.0', '1.0.0.RC1', '0.7.1', '0.7.0', '0.6.2', '0.6.1', '0.6.0', '0.5.1', '0.1.2', '0.1.1'], name: "apm_version", description: "APM Java Agent version")
-        choice(choices: ['adoptopenjdk-11+28-linux', 'adoptopenjdk-11.0.1+13-linux', 'adoptopenjdk-11.0.1+13-linux-aarch64', 'adoptopenjdk-11.0.2+7-linux', 'adoptopenjdk-11.0.2+7-linux-aarch64', 'adoptopenjdk-11.0.2+9-linux', 'adoptopenjdk-11.0.2+9-linux-aarch64', 'adoptopenjdk-11.0.3+7-linux', 'adoptopenjdk-11.0.3+7-linux-aarch64', 'adoptopenjdk-11.0.4+11-linux', 'adoptopenjdk-11.0.4+11-linux-aarch64', 'adoptopenjdk-11.0.5+10-linux', 'adoptopenjdk-11.0.6+10-linux', 'adoptopenjdk-11.0.6+10-linux-aarch64', 'adoptopenjdk-11.0.7+10-linux', 'adoptopenjdk-11.0.7+10-linux-aarch64', 'adoptopenjdk-11.0.8+10-linux', 'adoptopenjdk-11.0.8+10-linux-aarch64', 'adoptopenjdk-11.0.9+11-linux', 'adoptopenjdk-11.0.9+11-linux-aarch64', 'adoptopenjdk-12+33-linux', 'adoptopenjdk-12.0.1+12-linux', 'adoptopenjdk-12.0.1+12-linux-aarch64', 'adoptopenjdk-12.0.2+10-linux', 'adoptopenjdk-12.0.2+10-linux-aarch64', 'adoptopenjdk-13.0.1+9-linux', 'adoptopenjdk-13.0.2+8-linux', 'adoptopenjdk-13.0.2+8-linux-aarch64', 'adoptopenjdk-14.0.1+7-linux', 'adoptopenjdk-14.0.1+7-linux-aarch64', 'adoptopenjdk-14.0.2+12-linux', 'adoptopenjdk-14.0.2+12-linux-aarch64', 'adoptopenjdk-15+36-linux', 'adoptopenjdk-15+36-linux-aarch64', 'adoptopenjdk-15.0.1+9-linux', 'adoptopenjdk-15.0.1+9-linux-aarch64', 'openjdk-11+11-linux', 'openjdk-11+12-linux', 'openjdk-11+13-linux', 'openjdk-11+14-linux', 'openjdk-11+15-linux', 'openjdk-11+16-linux', 'openjdk-11+17-linux', 'openjdk-11+18-linux', 'openjdk-11+19-linux', 'openjdk-11+20-linux', 'openjdk-11+21-linux', 'openjdk-11+22-linux', 'openjdk-11+23-linux', 'openjdk-11+24-linux', 'openjdk-11+25-linux', 'openjdk-11+26-linux', 'openjdk-11+27-linux', 'openjdk-11+28-linux', 'openjdk-11+5-linux', 'openjdk-11-linux', 'openjdk-11.0.1-linux', 'openjdk-11.0.2-linux', 'openjdk-12+23-linux', 'openjdk-12+24-linux', 'openjdk-12+25-linux', 'openjdk-12+27-linux', 'openjdk-12+28-linux', 'openjdk-12+29-linux', 'openjdk-12+30-linux', 'openjdk-12+31-linux', 'openjdk-12+32-linux', 'openjdk-12+33-linux', 'openjdk-12-linux', 'openjdk-12.0.1-linux', 'openjdk-12.0.2-linux', 'openjdk-13+14-linux', 'openjdk-13+15-linux', 'openjdk-13+16-linux', 'openjdk-13+17-linux', 'openjdk-13+18-linux', 'openjdk-13+19-linux', 'openjdk-13+20-linux', 'openjdk-13+21-linux', 'openjdk-13+22-linux', 'openjdk-13+23-linux', 'openjdk-13+24-linux', 'openjdk-13+25-linux', 'openjdk-13+26-linux', 'openjdk-13+27-linux', 'openjdk-13+28-linux', 'openjdk-13+29-linux', 'openjdk-13+30-linux', 'openjdk-13+31-linux', 'openjdk-13+32-linux', 'openjdk-13-linux', 'openjdk-13.0.1-linux', 'openjdk-13.0.2-linux', 'openjdk-14+10-linux', 'openjdk-14+11-linux', 'openjdk-14+12-linux', 'openjdk-14+13-linux', 'openjdk-14+14-linux', 'openjdk-14+15-linux', 'openjdk-14+16-linux', 'openjdk-14+17-linux', 'openjdk-14+25-linux', 'openjdk-14+26-linux', 'openjdk-14+27-linux', 'openjdk-14+28-linux', 'openjdk-14+30-linux', 'openjdk-14+31-linux', 'openjdk-14+32-linux', 'openjdk-14+33-linux', 'openjdk-14+34-linux', 'openjdk-14+9-linux', 'openjdk-14-linux', 'openjdk-14.0.1-linux', 'openjdk-14.0.2+12-linux', 'openjdk-14.0.2-linux', 'openjdk-15+10-linux', 'openjdk-15+11-linux', 'openjdk-15+12-linux', 'openjdk-15+13-linux', 'openjdk-15+14-linux', 'openjdk-15+15-linux', 'openjdk-15+16-linux', 'openjdk-15+17-linux', 'openjdk-15+18-linux', 'openjdk-15+19-linux', 'openjdk-15+20-linux', 'openjdk-15+21-linux', 'openjdk-15+22-linux', 'openjdk-15+23-linux', 'openjdk-15+24-linux', 'openjdk-15+25-linux', 'openjdk-15+26-linux', 'openjdk-15+27-linux', 'openjdk-15+28-linux', 'openjdk-15+29-linux', 'openjdk-15+30-linux', 'openjdk-15+31-linux', 'openjdk-15+32-linux', 'openjdk-15+33-linux', 'openjdk-15+34-linux', 'openjdk-15+36-linux', 'openjdk-15+4-linux', 'openjdk-15+5-linux', 'openjdk-15+6-linux', 'openjdk-15+7-linux', 'openjdk-15+8-linux', 'openjdk-15+9-linux', 'openjdk-15-linux', 'openjdk-15.0.1+9-linux', 'oracle-11+11-linux', 'oracle-11+12-linux', 'oracle-11+13-linux', 'oracle-11+14-linux', 'oracle-11+15-linux', 'oracle-11+16-linux', 'oracle-11+17-linux', 'oracle-11+18-linux', 'oracle-11+19-linux', 'oracle-11+20-linux', 'oracle-11+21-linux', 'oracle-11+22-linux', 'oracle-11+23-linux', 'oracle-11+24-linux', 'oracle-11+25-linux', 'oracle-11+26-linux', 'oracle-11+27-linux', 'oracle-11+28-linux', 'oracle-11+5-linux', 'oracle-11.0.2+7-linux', 'oracle-11.0.2+9-linux', 'oracle-11.0.3+12-linux', 'oracle-11.0.4+10-linux', 'oracle-11.0.5+10-linux', 'oracle-11.0.6+8-linux', 'oracle-12+33-linux', 'oracle-12.0.1+12-linux', 'oracle-12.0.2+10-linux', 'oracle-13+33-linux', 'oracle-13.0.1+9-linux', 'oracle-13.0.2+8-linux', 'zulu-11.0.1-linux', 'zulu-11.0.2-linux', 'zulu-11.0.3-linux', 'zulu-11.0.4-linux', 'zulu-11.0.5-linux', 'zulu-11.0.6-linux', 'zulu-11.0.7-linux', 'zulu-11.0.8-linux', 'zulu-11.0.9-linux', 'zulu-12-linux', 'zulu-12.0.0-linux', 'zulu-12.0.1-linux', 'zulu-12.0.2-linux', 'zulu-13-linux', 'zulu-13.0.0-linux', 'zulu-13.0.1-linux', 'zulu-13.0.2-linux', 'zulu-13.0.3-linux', 'zulu-13.0.4-linux', 'zulu-13.0.5-linux', 'zulu-14-linux', 'zulu-14.0.0-linux', 'zulu-14.0.1-linux', 'zulu-14.0.2-linux', 'zulu-15.0.0-linux', 'zulu-15.0.1-linux'], name: "jvm_version", description: "JVM")
+        choice(choices: ['adoptopenjdk-11+28-linux', 'adoptopenjdk-11.0.1+13-linux', 'adoptopenjdk-11.0.2+7-linux', 'adoptopenjdk-11.0.2+9-linux', 'adoptopenjdk-11.0.3+7-linux', 'adoptopenjdk-11.0.4+11-linux', 'adoptopenjdk-11.0.5+10-linux', 'adoptopenjdk-11.0.6+10-linux', 'adoptopenjdk-11.0.7+10-linux', 'adoptopenjdk-11.0.8+10-linux', 'adoptopenjdk-11.0.9+11-linux', 'adoptopenjdk-12+33-linux', 'adoptopenjdk-12.0.1+12-linux', 'adoptopenjdk-12.0.2+10-linux', 'adoptopenjdk-13.0.1+9-linux', 'adoptopenjdk-13.0.2+8-linux', 'adoptopenjdk-14.0.1+7-linux', 'adoptopenjdk-14.0.2+12-linux', 'adoptopenjdk-15+36-linux', 'adoptopenjdk-15.0.1+9-linux', 'openjdk-10+43-linux', 'openjdk-10-linux', 'openjdk-10.0.1-linux', 'openjdk-10.0.2-linux', 'openjdk-11+11-linux', 'openjdk-11+12-linux', 'openjdk-11+13-linux', 'openjdk-11+14-linux', 'openjdk-11+15-linux', 'openjdk-11+16-linux', 'openjdk-11+17-linux', 'openjdk-11+18-linux', 'openjdk-11+19-linux', 'openjdk-11+20-linux', 'openjdk-11+21-linux', 'openjdk-11+22-linux', 'openjdk-11+23-linux', 'openjdk-11+24-linux', 'openjdk-11+25-linux', 'openjdk-11+26-linux', 'openjdk-11+27-linux', 'openjdk-11+28-linux', 'openjdk-11+5-linux', 'openjdk-11-linux', 'openjdk-11.0.1-linux', 'openjdk-11.0.2-linux', 'openjdk-12+23-linux', 'openjdk-12+24-linux', 'openjdk-12+25-linux', 'openjdk-12+27-linux', 'openjdk-12+28-linux', 'openjdk-12+29-linux', 'openjdk-12+30-linux', 'openjdk-12+31-linux', 'openjdk-12+32-linux', 'openjdk-12+33-linux', 'openjdk-12-linux', 'openjdk-12.0.1-linux', 'openjdk-12.0.2-linux', 'openjdk-13+14-linux', 'openjdk-13+15-linux', 'openjdk-13+16-linux', 'openjdk-13+17-linux', 'openjdk-13+18-linux', 'openjdk-13+19-linux', 'openjdk-13+20-linux', 'openjdk-13+21-linux', 'openjdk-13+22-linux', 'openjdk-13+23-linux', 'openjdk-13+24-linux', 'openjdk-13+25-linux', 'openjdk-13+26-linux', 'openjdk-13+27-linux', 'openjdk-13+28-linux', 'openjdk-13+29-linux', 'openjdk-13+30-linux', 'openjdk-13+31-linux', 'openjdk-13+32-linux', 'openjdk-13-linux', 'openjdk-13.0.1-linux', 'openjdk-13.0.2-linux', 'openjdk-14+10-linux', 'openjdk-14+11-linux', 'openjdk-14+12-linux', 'openjdk-14+13-linux', 'openjdk-14+14-linux', 'openjdk-14+15-linux', 'openjdk-14+16-linux', 'openjdk-14+17-linux', 'openjdk-14+25-linux', 'openjdk-14+26-linux', 'openjdk-14+27-linux', 'openjdk-14+28-linux', 'openjdk-14+30-linux', 'openjdk-14+31-linux', 'openjdk-14+32-linux', 'openjdk-14+33-linux', 'openjdk-14+34-linux', 'openjdk-14+9-linux', 'openjdk-14-linux', 'openjdk-14.0.1-linux', 'openjdk-14.0.2+12-linux', 'openjdk-14.0.2-linux', 'openjdk-15+10-linux', 'openjdk-15+11-linux', 'openjdk-15+12-linux', 'openjdk-15+13-linux', 'openjdk-15+14-linux', 'openjdk-15+15-linux', 'openjdk-15+16-linux', 'openjdk-15+17-linux', 'openjdk-15+18-linux', 'openjdk-15+19-linux', 'openjdk-15+20-linux', 'openjdk-15+21-linux', 'openjdk-15+22-linux', 'openjdk-15+23-linux', 'openjdk-15+24-linux', 'openjdk-15+25-linux', 'openjdk-15+26-linux', 'openjdk-15+27-linux', 'openjdk-15+28-linux', 'openjdk-15+29-linux', 'openjdk-15+30-linux', 'openjdk-15+31-linux', 'openjdk-15+32-linux', 'openjdk-15+33-linux', 'openjdk-15+34-linux', 'openjdk-15+36-linux', 'openjdk-15+4-linux', 'openjdk-15+5-linux', 'openjdk-15+6-linux', 'openjdk-15+7-linux', 'openjdk-15+8-linux', 'openjdk-15+9-linux', 'openjdk-15-linux', 'openjdk-15.0.1+9-linux', 'openjdk-9.0.4-linux', 'oracle-10+43-linux', 'oracle-10+46-linux', 'oracle-11+11-linux', 'oracle-11+12-linux', 'oracle-11+13-linux', 'oracle-11+14-linux', 'oracle-11+15-linux', 'oracle-11+16-linux', 'oracle-11+17-linux', 'oracle-11+18-linux', 'oracle-11+19-linux', 'oracle-11+20-linux', 'oracle-11+21-linux', 'oracle-11+22-linux', 'oracle-11+23-linux', 'oracle-11+24-linux', 'oracle-11+25-linux', 'oracle-11+26-linux', 'oracle-11+27-linux', 'oracle-11+28-linux', 'oracle-11+5-linux', 'oracle-11.0.2+7-linux', 'oracle-11.0.2+9-linux', 'oracle-11.0.3+12-linux', 'oracle-11.0.4+10-linux', 'oracle-11.0.5+10-linux', 'oracle-11.0.6+8-linux', 'oracle-12+33-linux', 'oracle-12.0.1+12-linux', 'oracle-12.0.2+10-linux', 'oracle-13+33-linux', 'oracle-13.0.1+9-linux', 'oracle-13.0.2+8-linux', 'oracle-9.0.4+11-linux', 'zulu-10.0.0-linux', 'zulu-10.0.1-linux', 'zulu-10.0.2-linux', 'zulu-11.0.1-linux', 'zulu-11.0.2-linux', 'zulu-11.0.3-linux', 'zulu-11.0.4-linux', 'zulu-11.0.5-linux', 'zulu-11.0.6-linux', 'zulu-11.0.7-linux', 'zulu-11.0.8-linux', 'zulu-11.0.9-linux', 'zulu-12-linux', 'zulu-12.0.0-linux', 'zulu-12.0.1-linux', 'zulu-12.0.2-linux', 'zulu-13-linux', 'zulu-13.0.0-linux', 'zulu-13.0.1-linux', 'zulu-13.0.2-linux', 'zulu-13.0.3-linux', 'zulu-13.0.4-linux', 'zulu-13.0.5-linux', 'zulu-14-linux', 'zulu-14.0.0-linux', 'zulu-14.0.1-linux', 'zulu-14.0.2-linux', 'zulu-15.0.0-linux', 'zulu-15.0.1-linux', 'zulu-7.0.101-linux', 'zulu-7.0.111-linux', 'zulu-7.0.121-linux', 'zulu-7.0.131-linux', 'zulu-7.0.141-linux', 'zulu-7.0.154-linux', 'zulu-7.0.161-linux', 'zulu-7.0.171-linux', 'zulu-7.0.181-linux', 'zulu-7.0.191-linux', 'zulu-7.0.201-linux', 'zulu-7.0.211-linux', 'zulu-7.0.222-linux', 'zulu-7.0.232-linux', 'zulu-7.0.242-linux', 'zulu-7.0.252-linux', 'zulu-7.0.262-linux', 'zulu-7.0.272-linux', 'zulu-7.0.282-linux', 'zulu-7.0.95-linux', 'zulu-8.0.102-linux', 'zulu-8.0.112-linux', 'zulu-8.0.121-linux', 'zulu-8.0.131-linux', 'zulu-8.0.144-linux', 'zulu-8.0.152-linux', 'zulu-8.0.162-linux', 'zulu-8.0.163-linux', 'zulu-8.0.172-linux', 'zulu-8.0.181-linux', 'zulu-8.0.192-linux', 'zulu-8.0.201-linux', 'zulu-8.0.202-linux', 'zulu-8.0.212-linux', 'zulu-8.0.222-linux', 'zulu-8.0.232-linux', 'zulu-8.0.242-linux', 'zulu-8.0.252-linux', 'zulu-8.0.262-linux', 'zulu-8.0.265-linux', 'zulu-8.0.272-linux', 'zulu-8.0.71-linux', 'zulu-8.0.72-linux', 'zulu-8.0.91-linux', 'zulu-8.0.92-linux', 'zulu-9.0.0-linux', 'zulu-9.0.1-linux', 'zulu-9.0.4-linux', 'zulu-9.0.7-linux'], name: "jvm_version", description: "JVM")
         string(name: "concurrent_requests", defaultValue: "100", description: "The number of concurrent requests to test with")
-        string(name: "duration", defaultValue: "10m", description: "Test duration in minutes. Max: 280 minutes")
+        string(name: "duration", defaultValue: "10", description: "Test duration in minutes. Max: 280")
         // num_of_runs currently unsupported
         // string(name: "num_of_runs", defaultValue: "1", description: "Number of test runs to execute")
         text(name: "agent_config", "defaultValue": "", description: "Custom APM Agent configuration. (WARNING: May echo to console. Do not supply sensitive data.)")
diff --git a/.ci/load/scripts/param_gen/gen_params.py b/.ci/load/scripts/param_gen/gen_params.py
index c989a8044f..89fef43cca 100755
--- a/.ci/load/scripts/param_gen/gen_params.py
+++ b/.ci/load/scripts/param_gen/gen_params.py
@@ -50,8 +50,13 @@
 from packaging.version import parse as parse_version
 
 CATALOG_URL = 'https://jvm-catalog.elastic.co'
-SUPPORTED_JDKS = ['oracle', 'openjdk', 'adoptopenjdk', 'amazon', 'jdk', 'zulu']
-MIN_JDK_VERSION = '11'
+# The limit of total choices must be < 256 or Jenkins will stacktrace.
+# The list below represents all choices currently in the catalog
+# *except* `amazon` and `jdk`.
+# See https://github.com/elastic/apm-agent-java/pull/1467#discussion_r516464187
+# for additional context and discussion.
+SUPPORTED_JDKS = ['oracle', 'openjdk', 'adoptopenjdk', 'zulu']
+MIN_JDK_VERSION = '7'
 
 parser = argparse.ArgumentParser(description="Jenkins JDK snippet generator")
 parser.add_argument(
@@ -78,8 +83,6 @@
 
 parsed_args = parser.parse_args()
 
-
-
 # Gather JDKs we can support
 r = requests.get(CATALOG_URL + '/jdks')
 if r.status_code != 200:
@@ -88,7 +91,7 @@
 supported_jdks = []
 
 for jdk in r.json():
-    # print(jdk)
+    arch = None
     try:
         flavor, ver, dist = jdk.split('-', 3)
     except ValueError:
@@ -97,6 +100,8 @@
     if dist in parsed_args.platforms and \
         flavor in SUPPORTED_JDKS and \
             parse_version(ver) >= parse_version(parsed_args.min_ver):
+        if arch:
+            continue
         supported_jdks.append(jdk)
 
 # Gather releases of the agent we can support
@@ -118,7 +123,7 @@
     'string(name: "duration", defaultValue: "10", description: "Test duration in minutes. Max: 280")',  # noqa E501
     '// num_of_runs currently unsupported',
     '// string(name: "num_of_runs", defaultValue: "1", description: "Number of test runs to execute")',  # noqa E501
-    'text(name: "agent_config", "defaultValue": "", description: "APM Agent configuration")',  # noqa E501
+    'text(name: "agent_config", "defaultValue": "", description: "Custom APM Agent configuration. (WARNING: May echo to console. Do not supply sensitive data.)")',  # noqa E501
     'text(name: "locustfile", "defaultValue": "", description: "Locust load-generator plan")',  # noqa E5011
     'booleanParam(name: "local_metrics", description: "Enable local metrics collection?", defaultValue: false)',  # noqa E501
     '// End script auto-generation',

From 7ce0452975eb6f1b4e5547558369d0b4b714f66f Mon Sep 17 00:00:00 2001
From: Mike Place <mike.place@elastic.co>
Date: Thu, 5 Nov 2020 15:42:43 +0100
Subject: [PATCH 35/37] Increase timeout

---
 .ci/load/Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.ci/load/Jenkinsfile b/.ci/load/Jenkinsfile
index f1d40d8289..573fcf5446 100644
--- a/.ci/load/Jenkinsfile
+++ b/.ci/load/Jenkinsfile
@@ -18,7 +18,7 @@ pipeline {
 
     }
     options {
-        timeout(time: 3, unit: 'HOURS')
+        timeout(time: 72, unit: 'HOURS')
         buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30'))
         timestamps()
         ansiColor('xterm')

From b76a251d87c5184de3ca7d644b6a7cd99548a4ba Mon Sep 17 00:00:00 2001
From: Mike Place <mike.place@elastic.co>
Date: Thu, 5 Nov 2020 16:23:54 +0100
Subject: [PATCH 36/37] Hide server URL

---
 .ci/load/Jenkinsfile | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.ci/load/Jenkinsfile b/.ci/load/Jenkinsfile
index 573fcf5446..b9d261da71 100644
--- a/.ci/load/Jenkinsfile
+++ b/.ci/load/Jenkinsfile
@@ -11,8 +11,6 @@ pipeline {
         // Set below for local development
         // ORCH_URL='10.0.2.2:8000'
         DEBUG_MODE = '0' // set to '0' for production
-        // This is a placeholder server. This will change to a dummy APM Server when it is available.
-        APM_SERVER_URL = 'https://2a2bd0e2806a47e5996eeeec6d22e6df.apm.eu-west-3.aws.elastic-cloud.com:443'
         LOCUST_RUN_TIME = "${params.duration}"
         LOCUST_USERS = "${params.concurrent_requests}"
 
@@ -140,7 +138,7 @@ pipeline {
                                 echo 'Starting test application in background..'
                                 dir("${APP_BASE_DIR}"){
                                     // Launch app in background
-                                    withSecretVault(secret: 'secret/apm-team/ci/apm-load-test-server', user_var_name: 'APM_TOKEN_TYPE', pass_var_name: 'ELASTIC_APM_API_KEY'){
+                                    withSecretVault(secret: 'secret/apm-team/ci/apm-load-test-server', user_var_name: 'APM_SERVER_URL', pass_var_name: 'ELASTIC_APM_API_KEY'){
                                         // Start with packaging things up
                                         sh(script: "./mvnw package")
                                         sh(script: "java -jar -javaagent:${WORKSPACE}/${AGENT_BASE_DIR}/apm-agent-java/elastic-apm-agent/target/elastic-apm-agent-${params.apm_version}.jar -Delastic.apm.server_urls=${env.APM_SERVER_URL} -Delastic.apm.secret_token=${env.ELASTIC_APM_API_KEY} -XX:+FlightRecorder -XX:StartFlightRecording=filename=flight.jfr ./target/spring-petclinic-*.jar &")

From 719b5851d7ed5f575618cb01ad0e16c0974d2bf7 Mon Sep 17 00:00:00 2001
From: Mike Place <mike.place@elastic.co>
Date: Thu, 5 Nov 2020 16:46:01 +0100
Subject: [PATCH 37/37] Switch app to benchmark label

---
 .ci/load/Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.ci/load/Jenkinsfile b/.ci/load/Jenkinsfile
index b9d261da71..8d135bec91 100644
--- a/.ci/load/Jenkinsfile
+++ b/.ci/load/Jenkinsfile
@@ -62,7 +62,7 @@ pipeline {
                     }
                 }
                 stage('Test application') {
-                    agent { label 'metal' }
+                    agent { label 'benchmark' }
                     stages{
                         stage('Provision Java') {
                             steps {