Add pipeline for flaky test runner job (elastic#46740)

* WIP Jenkinsfile for flaky test runner

* Fix syntax

* A few more jenkinsfile fixes

* A few more fixes

* Can't round numbers in Jenkins groovy sandbox apparently

* More fixes

* Only do build_kbn_tp_sample_panel_action once during flaky testing

* Fix path and try setting a different JOB value

* Keep track of flaky test failures

* Introduce flaky test for testing

* Some flaky test pipeline cleanup

* Fix a couple of issues with flaky test failure tracking

* Update flaky test runner build name/desc with metadata

* Revert "Introduce flaky test for testing"

This reverts commit 202e9d86cf130ef37aff1817781a5b694aa2aa32.

* Try adding a local shared library

* Move local library loading logic to kibana pipeline library

* Move shared groovy code to shared library

* Add missed file

* Add ability to specify multiple agents for flaky test runner

* Update kibana-pipeline-library version

* Fix bug causing early exit for oss ciGroup1

.ci/Jenkinsfile_flaky

@@ -0,0 +1,113 @@
+#!/bin/groovy
+
+library 'kibana-pipeline-library'
+kibanaLibrary.load()
+
+// Looks like 'oss:ciGroup:1' or 'oss:firefoxSmoke'
+def JOB_PARTS = params.CI_GROUP.split(':')
+def IS_XPACK = JOB_PARTS[0] == 'xpack'
+def JOB = JOB_PARTS[1]
+def CI_GROUP = JOB_PARTS[2] ?: ''
+
+def worker = getWorkerFromParams(IS_XPACK, JOB, CI_GROUP)
+
+def workerFailures = []
+
+currentBuild.displayName += trunc(" ${params.GITHUB_OWNER}:${params.branch_specifier}", 24)
+currentBuild.description = "${params.CI_GROUP}<br />Executions: ${params.NUMBER_EXECUTIONS}"
+
+// Note: If you increase agent count, it will execute NUMBER_EXECUTIONS per agent. It will not divide them up amongst the agents
+// e.g. NUMBER_EXECUTIONS = 25, agentCount = 4 results in 100 total executions
+def agentCount = 1
+
+stage("Kibana Pipeline") {
+  timeout(time: 180, unit: 'MINUTES') {
+    timestamps {
+      ansiColor('xterm') {
+        def agents = [:]
+        for(def agentNumber = 1; agentNumber <= agentCount; agentNumber++) {
+          agents["agent-${agentNumber}"] = {
+            catchError {
+              kibanaPipeline.withWorkers('flaky-test-runner', {
+                if (!IS_XPACK) {
+                  kibanaPipeline.buildOss()
+                  if (CI_GROUP == '1') {
+                    runbld "./test/scripts/jenkins_build_kbn_tp_sample_panel_action.sh"
+                  }
+                } else {
+                  kibanaPipeline.buildXpack()
+                }
+              }, getWorkerMap(agentNumber, params.NUMBER_EXECUTIONS.toInteger(), worker, workerFailures))()
+            }
+          }
+        }
+
+        parallel(agents)
+
+        currentBuild.description += ", Failures: ${workerFailures.size()}"
+
+        if (workerFailures.size() > 0) {
+          print "There were ${workerFailures.size()} test suite failures."
+          print "The executions that failed were:"
+          print workerFailures.join("\n")
+          print "Please check 'Test Result' and 'Pipeline Steps' pages for more info"
+        }
+      }
+    }
+  }
+}
+
+def getWorkerFromParams(isXpack, job, ciGroup) {
+  if (!isXpack) {
+    if (job == 'firefoxSmoke') {
+      return kibanaPipeline.getPostBuildWorker('firefoxSmoke', { runbld './test/scripts/jenkins_firefox_smoke.sh' })
+    } else if(job == 'visualRegression') {
+      return kibanaPipeline.getPostBuildWorker('visualRegression', { runbld './test/scripts/jenkins_visual_regression.sh' })
+    } else {
+      return kibanaPipeline.getOssCiGroupWorker(ciGroup)
+    }
+  }
+
+  if (job == 'firefoxSmoke') {
+    return kibanaPipeline.getPostBuildWorker('xpack-firefoxSmoke', { runbld './test/scripts/jenkins_xpack_firefox_smoke.sh' })
+  } else if(job == 'visualRegression') {
+    return kibanaPipeline.getPostBuildWorker('xpack-visualRegression', { runbld './test/scripts/jenkins_xpack_visual_regression.sh' })
+  } else {
+    return kibanaPipeline.getXpackCiGroupWorker(ciGroup)
+  }
+}
+
+def getWorkerMap(agentNumber, numberOfExecutions, worker, workerFailures, maxWorkers = 14) {
+  def workerMap = [:]
+  def numberOfWorkers = Math.min(numberOfExecutions, maxWorkers)
+
+  for(def i = 1; i <= numberOfWorkers; i++) {
+    def workerExecutions = numberOfExecutions/numberOfWorkers + (i <= numberOfExecutions%numberOfWorkers ? 1 : 0)
+
+    workerMap["agent-${agentNumber}-worker-${i}"] = { workerNumber ->
+      for(def j = 0; j < workerExecutions; j++) {
+        print "Execute agent-${agentNumber} worker-${workerNumber}: ${j}"
+        withEnv(["JOB=agent-${agentNumber}-worker-${workerNumber}-${j}"]) {
+          catchError {
+            try {
+              worker(workerNumber)
+            } catch (ex) {
+              workerFailures << "agent-${agentNumber} worker-${workerNumber}-${j}"
+              throw ex
+            }
+          }
+        }
+      }
+    }
+  }
+
+  return workerMap
+}
+
+def trunc(str, length) {
+  if (str.size() >= length) {
+    return str.take(length) + "..."
+  }
+
+  return str;
+}