Merge branch 'main' into ml-aiops-plugin-api

.buildkite/ftr_configs.yml

@@ -26,12 +26,14 @@ disabled:
   - x-pack/test/security_solution_cypress/cases_cli_config.ts
   - x-pack/test/security_solution_cypress/ccs_config.ts
   - x-pack/test/security_solution_cypress/cli_config.ts
+  - x-pack/test/security_solution_cypress/cli_config_parallel.ts
   - x-pack/test/security_solution_cypress/config.firefox.ts
   - x-pack/test/security_solution_cypress/config.ts
   - x-pack/test/security_solution_cypress/response_ops_cli_config.ts
   - x-pack/test/security_solution_cypress/upgrade_config.ts
   - x-pack/test/security_solution_cypress/visual_config.ts
   - x-pack/test/functional_enterprise_search/with_host_configured.config.ts
+  - x-pack/plugins/apm/ftr_e2e/ftr_config_open.ts
   - x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts
   - x-pack/plugins/apm/ftr_e2e/ftr_config.ts
 

.buildkite/pipelines/artifacts.yml

@@ -52,6 +52,17 @@ steps:
           limit: 1
 
   - command: KIBANA_DOCKER_CONTEXT=cloud .buildkite/scripts/steps/artifacts/docker_context.sh
+    label: 'Docker Context Verification'
+    agents:
+      queue: n2-2
+    timeout_in_minutes: 30
+    if: "build.env('RELEASE_BUILD') == null || build.env('RELEASE_BUILD') == '' || build.env('RELEASE_BUILD') == 'false'"
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: KIBANA_DOCKER_CONTEXT=ubi .buildkite/scripts/steps/artifacts/docker_context.sh
     label: 'Docker Context Verification'
     agents:
       queue: n2-2

.buildkite/pipelines/pull_request/security_solution.yml

@@ -5,6 +5,7 @@ steps:
       queue: ci-group-6
     depends_on: build
     timeout_in_minutes: 120
+    parallelism: 4
     retry:
       automatic:
         - exit_status: '*'

.buildkite/pull_requests.json

@@ -16,7 +16,25 @@
       "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))",
       "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))",
       "skip_ci_labels": ["skip-ci", "jenkins-ci"],
-      "skip_target_branches": ["6.8", "7.11", "7.12"]
+      "skip_target_branches": ["6.8", "7.11", "7.12"],
+      "skip_ci_on_only_changed": [
+        "^docs/",
+        "^rfcs/",
+        "^.ci/.+\\.yml$",
+        "^.ci/es-snapshots/",
+        "^.ci/pipeline-library/",
+        "^.ci/Jenkinsfile_[^/]+$",
+        "^\\.github/",
+        "\\.md$",
+        "^\\.backportrc\\.json$",
+        "^nav-kibana-dev\\.docnav\\.json$",
+        "^src/dev/prs/kibana_qa_pr_list\\.json$",
+        "^\\.buildkite/pull_requests\\.json$"
+      ],
+      "always_require_ci_on_changed": [
+        "^docs/developer/plugin-list.asciidoc$",
+        "/plugins/[^/]+/readme\\.(md|asciidoc)$"
+      ]
     }
   ]
 }

.buildkite/scripts/common/env.sh

@@ -38,6 +38,7 @@ export TEST_BROWSER_HEADLESS=1
 export ELASTIC_APM_ENVIRONMENT=ci
 export ELASTIC_APM_TRANSACTION_SAMPLE_RATE=0.1
 export ELASTIC_APM_SERVER_URL=https://kibana-ci-apm.apm.us-central1.gcp.cloud.es.io
+# Not really a secret, if APM supported public auth we would use it and APM requires that we use this name
 export ELASTIC_APM_SECRET_TOKEN=7YKhoXsO4MzjhXjx2c
 
 if is_pr; then

.buildkite/scripts/lifecycle/post_command.sh

@@ -26,6 +26,7 @@ if [[ "$IS_TEST_EXECUTION_STEP" == "true" ]]; then
   buildkite-agent artifact upload 'x-pack/test/functional/apps/reporting/reports/session/*.pdf'
   buildkite-agent artifact upload 'x-pack/test/functional/failure_debug/html/*.html'
   buildkite-agent artifact upload '.es/**/*.hprof'
+  buildkite-agent artifact upload 'data/es_debug_*.tar.gz'
 
   echo "--- Run Failed Test Reporter"
   node scripts/report_failed_tests --build-url="${BUILDKITE_BUILD_URL}#${BUILDKITE_JOB_ID}" 'target/junit/**/*.xml'

.buildkite/scripts/pipelines/pull_request/pipeline.js

@@ -9,14 +9,11 @@
 const execSync = require('child_process').execSync;
 const fs = require('fs');
 const { areChangesSkippable, doAnyChangesMatch } = require('kibana-buildkite-library');
-const { SKIPPABLE_PR_MATCHERS } = require('./skippable_pr_matchers');
-
-const REQUIRED_PATHS = [
-  // this file is auto-generated and changes to it need to be validated with CI
-  /^docs\/developer\/plugin-list.asciidoc$/,
-  // don't skip CI on prs with changes to plugin readme files /i is for case-insensitive matching
-  /\/plugins\/[^\/]+\/readme\.(md|asciidoc)$/i,
-];
+const prConfigs = require('../../../pull_requests.json');
+const prConfig = prConfigs.jobs.find((job) => job.pipelineSlug === 'kibana-pull-request');
+
+const REQUIRED_PATHS = prConfig.always_require_ci_on_changed.map((r) => new RegExp(r, 'i'));
+const SKIPPABLE_PR_MATCHERS = prConfig.skip_ci_on_only_changed.map((r) => new RegExp(r, 'i'));
 
 const getPipeline = (filename, removeSteps = true) => {
   const str = fs.readFileSync(filename).toString();

.buildkite/scripts/steps/artifacts/docker_context.sh

@@ -19,6 +19,8 @@ if [[ "$KIBANA_DOCKER_CONTEXT" == "default" ]]; then
   DOCKER_CONTEXT_FILE="kibana-$FULL_VERSION-docker-build-context.tar.gz"
 elif [[ "$KIBANA_DOCKER_CONTEXT" == "cloud" ]]; then
   DOCKER_CONTEXT_FILE="kibana-cloud-$FULL_VERSION-docker-build-context.tar.gz"
+elif [[ "$KIBANA_DOCKER_CONTEXT" == "ubi" ]]; then
+  DOCKER_CONTEXT_FILE="kibana-ubi8-$FULL_VERSION-docker-build-context.tar.gz"
 fi
 
 tar -xf "target/$DOCKER_CONTEXT_FILE" -C "$DOCKER_BUILD_FOLDER"

.buildkite/scripts/steps/code_coverage/jest_parallel.sh

@@ -2,8 +2,8 @@
 
 set -uo pipefail
 
-JOB=$BUILDKITE_PARALLEL_JOB
-JOB_COUNT=$BUILDKITE_PARALLEL_JOB_COUNT
+JOB=${BUILDKITE_PARALLEL_JOB:-0}
+JOB_COUNT=${BUILDKITE_PARALLEL_JOB_COUNT:-1}
 
 # a jest failure will result in the script returning an exit code of 10
 
@@ -35,4 +35,4 @@ while read -r config; do
 # uses heredoc to avoid the while loop being in a sub-shell thus unable to overwrite exitCode
 done <<< "$(find src x-pack packages -name jest.config.js -not -path "*/__fixtures__/*" | sort)"
 
-exit $exitCode
+exit $exitCode

.buildkite/scripts/steps/es_snapshots/build.sh

@@ -69,7 +69,6 @@ echo "--- Build Elasticsearch"
   :distribution:archives:darwin-aarch64-tar:assemble \
   :distribution:archives:darwin-tar:assemble \
   :distribution:docker:docker-export:assemble \
-  :distribution:docker:cloud-docker-export:assemble \
   :distribution:archives:linux-aarch64-tar:assemble \
   :distribution:archives:linux-tar:assemble \
   :distribution:archives:windows-zip:assemble \
@@ -86,19 +85,26 @@ docker images "docker.elastic.co/elasticsearch/elasticsearch" --format "{{.Tag}}
 docker images "docker.elastic.co/elasticsearch/elasticsearch" --format "{{.Tag}}" | xargs -n1 bash -c 'docker save docker.elastic.co/elasticsearch/elasticsearch:${0} | gzip > ../es-build/elasticsearch-${0}-docker-image.tar.gz'
 
 echo "--- Create kibana-ci docker cloud image archives"
-ES_CLOUD_ID=$(docker images "docker.elastic.co/elasticsearch-ci/elasticsearch-cloud" --format "{{.ID}}")
-ES_CLOUD_VERSION=$(docker images "docker.elastic.co/elasticsearch-ci/elasticsearch-cloud" --format "{{.Tag}}")
-KIBANA_ES_CLOUD_VERSION="$ES_CLOUD_VERSION-$ELASTICSEARCH_GIT_COMMIT"
-KIBANA_ES_CLOUD_IMAGE="docker.elastic.co/kibana-ci/elasticsearch-cloud:$KIBANA_ES_CLOUD_VERSION"
-
-docker tag "$ES_CLOUD_ID" "$KIBANA_ES_CLOUD_IMAGE"
-
-echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
-trap 'docker logout docker.elastic.co' EXIT
-docker image push "$KIBANA_ES_CLOUD_IMAGE"
-
-export ELASTICSEARCH_CLOUD_IMAGE="$KIBANA_ES_CLOUD_IMAGE"
-export ELASTICSEARCH_CLOUD_IMAGE_CHECKSUM="$(docker images "$KIBANA_ES_CLOUD_IMAGE" --format "{{.Digest}}")"
+# Ignore build failures.  This docker image downloads metricbeat and filebeat.
+# When we bump versions, these dependencies may not exist yet, but we don't want to
+# block the rest of the snapshot promotion process
+set +e
+./gradlew :distribution:docker:cloud-docker-export:assemble && {
+  ES_CLOUD_ID=$(docker images "docker.elastic.co/elasticsearch-ci/elasticsearch-cloud" --format "{{.ID}}")
+  ES_CLOUD_VERSION=$(docker images "docker.elastic.co/elasticsearch-ci/elasticsearch-cloud" --format "{{.Tag}}")
+  KIBANA_ES_CLOUD_VERSION="$ES_CLOUD_VERSION-$ELASTICSEARCH_GIT_COMMIT"
+  KIBANA_ES_CLOUD_IMAGE="docker.elastic.co/kibana-ci/elasticsearch-cloud:$KIBANA_ES_CLOUD_VERSION"
+  echo $ES_CLOUD_ID $ES_CLOUD_VERSION $KIBANA_ES_CLOUD_VERSION $KIBANA_ES_CLOUD_IMAGE
+  docker tag "$ES_CLOUD_ID" "$KIBANA_ES_CLOUD_IMAGE"
+
+  echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
+  trap 'docker logout docker.elastic.co' EXIT
+  docker image push "$KIBANA_ES_CLOUD_IMAGE"
+
+  export ELASTICSEARCH_CLOUD_IMAGE="$KIBANA_ES_CLOUD_IMAGE"
+  export ELASTICSEARCH_CLOUD_IMAGE_CHECKSUM="$(docker images "$KIBANA_ES_CLOUD_IMAGE" --format "{{.Digest}}")"
+}
+set -e
 
 echo "--- Create checksums for snapshot files"
 cd "$destination"

.buildkite/scripts/steps/functional/security_solution.sh

@@ -5,11 +5,13 @@ set -euo pipefail
 source .buildkite/scripts/steps/functional/common.sh
 
 export JOB=kibana-security-solution-chrome
+export CLI_NUMBER=$((BUILDKITE_PARALLEL_JOB+1))
+export CLI_COUNT=$BUILDKITE_PARALLEL_JOB_COUNT
 
 echo "--- Security Solution tests (Chrome)"
 
-checks-reporter-with-killswitch "Security Solution Cypress Tests (Chrome)" \
+checks-reporter-with-killswitch "Security Solution Cypress Tests (Chrome) $CLI_NUMBER" \
  node scripts/functional_tests \
    --debug --bail \
    --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
-   --config x-pack/test/security_solution_cypress/cli_config.ts
+   --config x-pack/test/security_solution_cypress/cli_config_parallel.ts

.buildkite/scripts/steps/test/ftr_configs.sh

@@ -4,10 +4,10 @@ set -euo pipefail
 
 source .buildkite/scripts/steps/functional/common.sh
 
-export JOB_NUM=$BUILDKITE_PARALLEL_JOB
+export JOB_NUM=${BUILDKITE_PARALLEL_JOB:-0}
 export JOB=ftr-configs-${JOB_NUM}
 
-FAILED_CONFIGS_KEY="${BUILDKITE_STEP_ID}${BUILDKITE_PARALLEL_JOB:-0}"
+FAILED_CONFIGS_KEY="${BUILDKITE_STEP_ID}${JOB_NUM}"
 
 # a FTR failure will result in the script returning an exit code of 10
 exitCode=0

.buildkite/scripts/steps/test/jest.sh

@@ -8,6 +8,8 @@ is_test_execution_step
 
 .buildkite/scripts/bootstrap.sh
 
+JOB=${BUILDKITE_PARALLEL_JOB:-0}
+
 echo '--- Jest'
-checks-reporter-with-killswitch "Jest Unit Tests $((BUILDKITE_PARALLEL_JOB+1))" \
+checks-reporter-with-killswitch "Jest Unit Tests $((JOB+1))" \
   .buildkite/scripts/steps/test/jest_parallel.sh jest.config.js

.buildkite/scripts/steps/test/jest_integration.sh

@@ -8,6 +8,8 @@ is_test_execution_step
 
 .buildkite/scripts/bootstrap.sh
 
+JOB=${BUILDKITE_PARALLEL_JOB:-0}
+
 echo '--- Jest Integration Tests'
-checks-reporter-with-killswitch "Jest Integration Tests $((BUILDKITE_PARALLEL_JOB+1))" \
+checks-reporter-with-killswitch "Jest Integration Tests $((JOB+1))" \
   .buildkite/scripts/steps/test/jest_parallel.sh jest.integration.config.js

.buildkite/scripts/steps/test/jest_parallel.sh

@@ -2,7 +2,7 @@
 
 set -euo pipefail
 
-export JOB=$BUILDKITE_PARALLEL_JOB
+export JOB=${BUILDKITE_PARALLEL_JOB:-0}
 
 # a jest failure will result in the script returning an exit code of 10
 exitCode=0