Merge branch 'main' of github.com:elastic/kibana into 12-04-2022-dela…

…y-journey-finish

.bazelrc

@@ -2,14 +2,6 @@
 # Import shared settings first so we can override below
 import %workspace%/.bazelrc.common
 
-# Remote cache settings for local env
-# build --remote_cache=grpcs://cloud.buildbuddy.io
-# build --incompatible_remote_results_ignore_disk=true
-# build --noremote_upload_local_results
-# build --remote_timeout=30
-# build --remote_header=x-buildbuddy-api-key=3EYk49W2NefOx2n3yMze
-# build --remote_accept_cached=true
-
 # Enable this in case you want to share your build info
 # build --build_metadata=VISIBILITY=PUBLIC
 build --build_metadata=TEST_GROUPS=//packages

.buildkite/pipelines/artifacts.yml

@@ -4,29 +4,68 @@ steps:
     agents:
       queue: c2-16
     timeout_in_minutes: 60
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
 
   - wait
 
   - command: TEST_PACKAGE=deb .buildkite/scripts/steps/package_testing/test.sh
-    label: Package testing for deb
+    label: Artifact Testing
     agents:
       queue: n2-4-virt
-    timeout_in_minutes: 20
+    timeout_in_minutes: 30
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
 
   - command: TEST_PACKAGE=rpm .buildkite/scripts/steps/package_testing/test.sh
-    label: Package testing for rpm
+    label: Artifact Testing
     agents:
       queue: n2-4-virt
-    timeout_in_minutes: 20
+    timeout_in_minutes: 30
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
 
   - command: TEST_PACKAGE=docker .buildkite/scripts/steps/package_testing/test.sh
-    label: Package testing for docker
+    label: Artifact Testing
     agents:
       queue: n2-4-virt
-    timeout_in_minutes: 20
+    timeout_in_minutes: 30
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
 
   - command: .buildkite/scripts/steps/artifacts/docker_context.sh
-    label: 'Docker Build Context'
+    label: 'Docker Context Verification'
     agents:
       queue: n2-2
-    timeout_in_minutes: 20
+    timeout_in_minutes: 30
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/artifacts/cloud.sh
+    label: 'Cloud Deployment'
+    agents:
+      queue: n2-2
+    timeout_in_minutes: 30
+    if: "build.env('RELEASE_BUILD') == null || build.env('RELEASE_BUILD') == '' || build.env('RELEASE_BUILD') == 'false'"
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - wait
+
+  - command: .buildkite/scripts/steps/artifacts/publish.sh
+    label: 'Publish Kibana Artifacts'
+    agents:
+      queue: n2-2
+    timeout_in_minutes: 30

.buildkite/pipelines/es_snapshots/verify.yml

@@ -29,7 +29,7 @@ steps:
 
   - command: .buildkite/scripts/steps/functional/xpack_cigroup.sh
     label: 'Default CI Group'
-    parallelism: 27
+    parallelism: 30
     agents:
       queue: n2-4
     depends_on: build
@@ -40,18 +40,6 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - command: CI_GROUP=Docker .buildkite/scripts/steps/functional/xpack_cigroup.sh
-    label: 'Docker CI Group'
-    agents:
-      queue: n2-4
-    depends_on: build
-    timeout_in_minutes: 120
-    key: default-cigroup-docker
-    retry:
-      automatic:
-        - exit_status: '*'
-          limit: 1
-
   - command: .buildkite/scripts/steps/functional/oss_cigroup.sh
     label: 'OSS CI Group'
     parallelism: 11

.buildkite/pipelines/flaky_tests/groups.json

@@ -26,13 +26,13 @@
       "name": "Fleet - Cypress"
     },
     {
-      "key": "xpack/cigroup",
-      "name": "Default CI Group",
-      "ciGroups": 27
+      "key": "xpack/cypress/apm_cypress",
+      "name": "APM - Cypress"
     },
     {
-      "key": "xpack/cigroup/Docker",
-      "name": "Default CI Group Docker"
+      "key": "xpack/cigroup",
+      "name": "Default CI Group",
+      "ciGroups": 30
     },
     {
       "key": "xpack/firefox",

.buildkite/pipelines/on_merge.yml

@@ -51,7 +51,7 @@ steps:
 
   - command: .buildkite/scripts/steps/functional/xpack_cigroup.sh
     label: 'Default CI Group'
-    parallelism: 27
+    parallelism: 30
     agents:
       queue: n2-4-spot
     depends_on: build
@@ -64,20 +64,6 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - command: CI_GROUP=Docker .buildkite/scripts/steps/functional/xpack_cigroup.sh
-    label: 'Docker CI Group'
-    agents:
-      queue: n2-4-spot
-    depends_on: build
-    timeout_in_minutes: 120
-    key: default-cigroup-docker
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 3
-        - exit_status: '*'
-          limit: 1
-
   - command: .buildkite/scripts/steps/functional/oss_cigroup.sh
     label: 'OSS CI Group'
     parallelism: 11

.buildkite/pipelines/pull_request/base.yml

@@ -17,39 +17,31 @@ steps:
 
   - command: .buildkite/scripts/steps/functional/xpack_cigroup.sh
     label: 'Default CI Group'
-    parallelism: 27
+    parallelism: 30
     agents:
-      queue: n2-4
+      queue: n2-4-spot-2
     depends_on: build
     timeout_in_minutes: 150
     key: default-cigroup
     retry:
       automatic:
-        - exit_status: '*'
-          limit: 1
-
-  - command: CI_GROUP=Docker .buildkite/scripts/steps/functional/xpack_cigroup.sh
-    label: 'Docker CI Group'
-    agents:
-      queue: n2-4
-    depends_on: build
-    timeout_in_minutes: 120
-    key: default-cigroup-docker
-    retry:
-      automatic:
+        - exit_status: '-1'
+          limit: 3
         - exit_status: '*'
           limit: 1
 
   - command: .buildkite/scripts/steps/functional/oss_cigroup.sh
     label: 'OSS CI Group'
     parallelism: 11
     agents:
-      queue: ci-group-4d
+      queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 120
     key: oss-cigroup
     retry:
       automatic:
+        - exit_status: '-1'
+          limit: 3
         - exit_status: '*'
           limit: 1
 
@@ -61,6 +53,8 @@ steps:
     timeout_in_minutes: 120
     retry:
       automatic:
+        - exit_status: '-1'
+          limit: 3
         - exit_status: '*'
           limit: 1
 
@@ -72,6 +66,8 @@ steps:
     timeout_in_minutes: 120
     retry:
       automatic:
+        - exit_status: '-1'
+          limit: 3
         - exit_status: '*'
           limit: 1
 
@@ -83,6 +79,8 @@ steps:
     timeout_in_minutes: 120
     retry:
       automatic:
+        - exit_status: '-1'
+          limit: 3
         - exit_status: '*'
           limit: 1
 
@@ -94,6 +92,8 @@ steps:
     timeout_in_minutes: 120
     retry:
       automatic:
+        - exit_status: '-1'
+          limit: 3
         - exit_status: '*'
           limit: 1
 
@@ -105,6 +105,8 @@ steps:
     timeout_in_minutes: 120
     retry:
       automatic:
+        - exit_status: '-1'
+          limit: 3
         - exit_status: '*'
           limit: 1
 
@@ -116,31 +118,45 @@ steps:
     timeout_in_minutes: 120
     retry:
       automatic:
+        - exit_status: '-1'
+          limit: 3
         - exit_status: '*'
           limit: 1
 
   - command: .buildkite/scripts/steps/test/jest.sh
     label: 'Jest Tests'
     parallelism: 8
     agents:
-      queue: n2-4
+      queue: n2-4-spot
     timeout_in_minutes: 90
     key: jest
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
 
   - command: .buildkite/scripts/steps/test/jest_integration.sh
     label: 'Jest Integration Tests'
     parallelism: 3
     agents:
-      queue: n2-4
+      queue: n2-4-spot
     timeout_in_minutes: 120
     key: jest-integration
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
 
   - command: .buildkite/scripts/steps/test/api_integration.sh
     label: 'API Integration Tests'
     agents:
       queue: n2-2-spot
     timeout_in_minutes: 120
     key: api-integration
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
 
   - command: .buildkite/scripts/steps/lint.sh
     label: 'Linting'
@@ -176,6 +192,10 @@ steps:
       queue: n2-4-spot
     key: build_api_docs
     timeout_in_minutes: 60
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
 
   - command: .buildkite/scripts/steps/webpack_bundle_analyzer/build_and_upload.sh
     label: 'Build Webpack Bundle Analyzer reports'

.buildkite/pipelines/pull_request/response_ops.yml

@@ -0,0 +1,11 @@
+steps:
+  - command: .buildkite/scripts/steps/functional/response_ops_cases.sh
+    label: 'Cases Cypress Tests on Security Solution'
+    agents:
+      queue: ci-group-6
+    depends_on: build
+    timeout_in_minutes: 120
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1

.buildkite/scripts/build_kibana_plugins.sh

@@ -3,18 +3,4 @@
 set -euo pipefail
 
 echo "--- Build Platform Plugins"
-node scripts/build_kibana_platform_plugins \
-  --scan-dir "$KIBANA_DIR/test/analytics/__fixtures__/plugins" \
-  --scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \
-  --scan-dir "$KIBANA_DIR/test/interpreter_functional/plugins" \
-  --scan-dir "$KIBANA_DIR/test/common/fixtures/plugins" \
-  --scan-dir "$KIBANA_DIR/examples" \
-  --scan-dir "$XPACK_DIR/test/plugin_functional/plugins" \
-  --scan-dir "$XPACK_DIR/test/functional_with_es_ssl/fixtures/plugins" \
-  --scan-dir "$XPACK_DIR/test/alerting_api_integration/plugins" \
-  --scan-dir "$XPACK_DIR/test/plugin_api_integration/plugins" \
-  --scan-dir "$XPACK_DIR/test/plugin_api_perf/plugins" \
-  --scan-dir "$XPACK_DIR/test/licensing_plugin/plugins" \
-  --scan-dir "$XPACK_DIR/test/usage_collection/plugins" \
-  --scan-dir "$XPACK_DIR/test/security_functional/fixtures/common" \
-  --scan-dir "$XPACK_DIR/examples"
+node scripts/build_kibana_platform_plugins --examples --test-plugins

.buildkite/scripts/common/setup_bazel.sh

@@ -15,8 +15,8 @@ if [[ "${BAZEL_CACHE_MODE:-none}" == read* ]]; then
   echo "[bazel] enabling caching"
 cat <<EOF >> $KIBANA_DIR/.bazelrc
   build --bes_results_url=https://app.buildbuddy.io/invocation/
-  build --bes_backend=grpcs://cloud.buildbuddy.io
-  build --remote_cache=grpcs://cloud.buildbuddy.io
+  build --bes_backend=grpcs://remote.buildbuddy.io
+  build --remote_cache=grpcs://remote.buildbuddy.io
   build --remote_timeout=3600
   build --remote_header=x-buildbuddy-api-key=$KIBANA_BUILDBUDDY_CI_API_KEY
 EOF

.buildkite/scripts/lifecycle/pre_build.sh

@@ -12,3 +12,11 @@ export CI_STATS_TOKEN="$(retry 5 5 vault read -field=api_token secret/kibana-iss
 export CI_STATS_HOST="$(retry 5 5 vault read -field=api_host secret/kibana-issues/dev/kibana_ci_stats)"
 
 node "$(dirname "${0}")/ci_stats_start.js"
+
+# We resolve the latest manifest URL at the beginning of the build to ensure that all steps in the build will use the same manifest
+# Otherwise, the manifest could change if a step is running around the time that a new one is promoted
+if [[ ! "${ES_SNAPSHOT_MANIFEST:-}" ]]; then
+  BUCKET=$(curl -s "https://storage.googleapis.com/kibana-ci-es-snapshots-daily/$(cat package.json | jq -r .version)/manifest-latest-verified.json" | jq -r .bucket)
+  ES_SNAPSHOT_MANIFEST_DEFAULT="https://storage.googleapis.com/$BUCKET/manifest.json"
+  buildkite-agent meta-data set ES_SNAPSHOT_MANIFEST_DEFAULT "$ES_SNAPSHOT_MANIFEST_DEFAULT"
+fi

.buildkite/scripts/lifecycle/pre_command.sh

@@ -65,6 +65,15 @@ EOF
   fi
 }
 
+# If a custom manifest isn't specified, then use the default one that we resolve earlier in the build
+{
+  if [[ ! "${ES_SNAPSHOT_MANIFEST:-}" ]]; then
+    ES_SNAPSHOT_MANIFEST=${ES_SNAPSHOT_MANIFEST:-$(buildkite-agent meta-data get ES_SNAPSHOT_MANIFEST_DEFAULT --default '')}
+    export ES_SNAPSHOT_MANIFEST
+    echo "Using default ES Snapshot Manifest: $ES_SNAPSHOT_MANIFEST"
+  fi
+}
+
 # Setup CI Stats
 {
   CI_STATS_BUILD_ID="$(buildkite-agent meta-data get ci_stats_build_id --default '')"