Merge branch '8.2' of https://github.com/elastic/kibana into backport…

…/8.2/pr-130123

.bazelrc

@@ -2,14 +2,6 @@
 # Import shared settings first so we can override below
 import %workspace%/.bazelrc.common
 
-# Remote cache settings for local env
-# build --remote_cache=grpcs://cloud.buildbuddy.io
-# build --incompatible_remote_results_ignore_disk=true
-# build --noremote_upload_local_results
-# build --remote_timeout=30
-# build --remote_header=x-buildbuddy-api-key=3EYk49W2NefOx2n3yMze
-# build --remote_accept_cached=true
-
 # Enable this in case you want to share your build info
 # build --build_metadata=VISIBILITY=PUBLIC
 build --build_metadata=TEST_GROUPS=//packages

.buildkite/pipelines/es_snapshots/verify.yml

@@ -29,7 +29,7 @@ steps:
 
   - command: .buildkite/scripts/steps/functional/xpack_cigroup.sh
     label: 'Default CI Group'
-    parallelism: 27
+    parallelism: 29
     agents:
       queue: n2-4
     depends_on: build
@@ -40,18 +40,6 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - command: CI_GROUP=Docker .buildkite/scripts/steps/functional/xpack_cigroup.sh
-    label: 'Docker CI Group'
-    agents:
-      queue: n2-4
-    depends_on: build
-    timeout_in_minutes: 120
-    key: default-cigroup-docker
-    retry:
-      automatic:
-        - exit_status: '*'
-          limit: 1
-
   - command: .buildkite/scripts/steps/functional/oss_cigroup.sh
     label: 'OSS CI Group'
     parallelism: 11

.buildkite/pipelines/flaky_tests/groups.json

@@ -28,11 +28,7 @@
     {
       "key": "xpack/cigroup",
       "name": "Default CI Group",
-      "ciGroups": 27
-    },
-    {
-      "key": "xpack/cigroup/Docker",
-      "name": "Default CI Group Docker"
+      "ciGroups": 29
     },
     {
       "key": "xpack/firefox",

.buildkite/pipelines/on_merge.yml

@@ -51,7 +51,7 @@ steps:
 
   - command: .buildkite/scripts/steps/functional/xpack_cigroup.sh
     label: 'Default CI Group'
-    parallelism: 27
+    parallelism: 29
     agents:
       queue: n2-4-spot
     depends_on: build
@@ -64,20 +64,6 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - command: CI_GROUP=Docker .buildkite/scripts/steps/functional/xpack_cigroup.sh
-    label: 'Docker CI Group'
-    agents:
-      queue: n2-4-spot
-    depends_on: build
-    timeout_in_minutes: 120
-    key: default-cigroup-docker
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 3
-        - exit_status: '*'
-          limit: 1
-
   - command: .buildkite/scripts/steps/functional/oss_cigroup.sh
     label: 'OSS CI Group'
     parallelism: 11

.buildkite/pipelines/pull_request/base.yml

@@ -17,7 +17,7 @@ steps:
 
   - command: .buildkite/scripts/steps/functional/xpack_cigroup.sh
     label: 'Default CI Group'
-    parallelism: 27
+    parallelism: 29
     agents:
       queue: n2-4
     depends_on: build
@@ -28,28 +28,18 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - command: CI_GROUP=Docker .buildkite/scripts/steps/functional/xpack_cigroup.sh
-    label: 'Docker CI Group'
-    agents:
-      queue: n2-4
-    depends_on: build
-    timeout_in_minutes: 120
-    key: default-cigroup-docker
-    retry:
-      automatic:
-        - exit_status: '*'
-          limit: 1
-
   - command: .buildkite/scripts/steps/functional/oss_cigroup.sh
     label: 'OSS CI Group'
     parallelism: 11
     agents:
-      queue: ci-group-4d
+      queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 120
     key: oss-cigroup
     retry:
       automatic:
+        - exit_status: '-1'
+          limit: 3
         - exit_status: '*'
           limit: 1
 
@@ -61,6 +51,8 @@ steps:
     timeout_in_minutes: 120
     retry:
       automatic:
+        - exit_status: '-1'
+          limit: 3
         - exit_status: '*'
           limit: 1
 
@@ -72,6 +64,8 @@ steps:
     timeout_in_minutes: 120
     retry:
       automatic:
+        - exit_status: '-1'
+          limit: 3
         - exit_status: '*'
           limit: 1
 
@@ -83,6 +77,8 @@ steps:
     timeout_in_minutes: 120
     retry:
       automatic:
+        - exit_status: '-1'
+          limit: 3
         - exit_status: '*'
           limit: 1
 
@@ -94,6 +90,8 @@ steps:
     timeout_in_minutes: 120
     retry:
       automatic:
+        - exit_status: '-1'
+          limit: 3
         - exit_status: '*'
           limit: 1
 
@@ -105,6 +103,8 @@ steps:
     timeout_in_minutes: 120
     retry:
       automatic:
+        - exit_status: '-1'
+          limit: 3
         - exit_status: '*'
           limit: 1
 
@@ -116,31 +116,45 @@ steps:
     timeout_in_minutes: 120
     retry:
       automatic:
+        - exit_status: '-1'
+          limit: 3
         - exit_status: '*'
           limit: 1
 
   - command: .buildkite/scripts/steps/test/jest.sh
     label: 'Jest Tests'
     parallelism: 8
     agents:
-      queue: n2-4
+      queue: n2-4-spot
     timeout_in_minutes: 90
     key: jest
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
 
   - command: .buildkite/scripts/steps/test/jest_integration.sh
     label: 'Jest Integration Tests'
     parallelism: 3
     agents:
-      queue: n2-4
+      queue: n2-4-spot
     timeout_in_minutes: 120
     key: jest-integration
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
 
   - command: .buildkite/scripts/steps/test/api_integration.sh
     label: 'API Integration Tests'
     agents:
       queue: n2-2
     timeout_in_minutes: 120
     key: api-integration
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
 
   - command: .buildkite/scripts/steps/lint.sh
     label: 'Linting'
@@ -176,6 +190,10 @@ steps:
       queue: n2-4
     key: build_api_docs
     timeout_in_minutes: 60
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
 
   - command: .buildkite/scripts/steps/webpack_bundle_analyzer/build_and_upload.sh
     label: 'Build Webpack Bundle Analyzer reports'

.buildkite/scripts/common/setup_bazel.sh

@@ -15,8 +15,8 @@ if [[ "${BAZEL_CACHE_MODE:-none}" == read* ]]; then
   echo "[bazel] enabling caching"
 cat <<EOF >> $KIBANA_DIR/.bazelrc
   build --bes_results_url=https://app.buildbuddy.io/invocation/
-  build --bes_backend=grpcs://cloud.buildbuddy.io
-  build --remote_cache=grpcs://cloud.buildbuddy.io
+  build --bes_backend=grpcs://remote.buildbuddy.io
+  build --remote_cache=grpcs://remote.buildbuddy.io
   build --remote_timeout=3600
   build --remote_header=x-buildbuddy-api-key=$KIBANA_BUILDBUDDY_CI_API_KEY
 EOF

.buildkite/scripts/lifecycle/pre_build.sh

@@ -12,3 +12,11 @@ export CI_STATS_TOKEN="$(retry 5 5 vault read -field=api_token secret/kibana-iss
 export CI_STATS_HOST="$(retry 5 5 vault read -field=api_host secret/kibana-issues/dev/kibana_ci_stats)"
 
 node "$(dirname "${0}")/ci_stats_start.js"
+
+# We resolve the latest manifest URL at the beginning of the build to ensure that all steps in the build will use the same manifest
+# Otherwise, the manifest could change if a step is running around the time that a new one is promoted
+if [[ ! "${ES_SNAPSHOT_MANIFEST:-}" ]]; then
+  BUCKET=$(curl -s "https://storage.googleapis.com/kibana-ci-es-snapshots-daily/$(cat package.json | jq -r .version)/manifest-latest-verified.json" | jq -r .bucket)
+  ES_SNAPSHOT_MANIFEST_DEFAULT="https://storage.googleapis.com/$BUCKET/manifest.json"
+  buildkite-agent meta-data set ES_SNAPSHOT_MANIFEST_DEFAULT "$ES_SNAPSHOT_MANIFEST_DEFAULT"
+fi

.buildkite/scripts/lifecycle/pre_command.sh

@@ -65,6 +65,15 @@ EOF
   fi
 }
 
+# If a custom manifest isn't specified, then use the default one that we resolve earlier in the build
+{
+  if [[ ! "${ES_SNAPSHOT_MANIFEST:-}" ]]; then
+    ES_SNAPSHOT_MANIFEST=${ES_SNAPSHOT_MANIFEST:-$(buildkite-agent meta-data get ES_SNAPSHOT_MANIFEST_DEFAULT --default '')}
+    export ES_SNAPSHOT_MANIFEST
+    echo "Using default ES Snapshot Manifest: $ES_SNAPSHOT_MANIFEST"
+  fi
+}
+
 # Setup CI Stats
 {
   CI_STATS_BUILD_ID="$(buildkite-agent meta-data get ci_stats_build_id --default '')"

.buildkite/scripts/steps/artifacts/build.sh

@@ -9,9 +9,18 @@ source .buildkite/scripts/steps/artifacts/env.sh
 echo "--- Build Kibana artifacts"
 node scripts/build --all-platforms --debug --docker-cross-compile $(echo "$BUILD_ARGS")
 
-echo "--- Build and upload dependencies report"
+echo "--- Extract default i18n messages"
+mkdir -p target/i18n
+node scripts/i18n_extract --output-dir=target/i18n
+
+echo "--- Build dependencies report"
 node scripts/licenses_csv_report "--csv=target/dependencies-$FULL_VERSION.csv"
+(cd target; sha512sum "dependencies-$FULL_VERSION.csv" > "dependencies-$FULL_VERSION.csv.sha512.txt")
+
+echo "--- Upload Kibana Artifacts"
 cd target
-sha512sum "dependencies-$FULL_VERSION.csv" > "dependencies-$FULL_VERSION.csv.sha512.txt"
-buildkite-agent artifact upload "*"
+buildkite-agent artifact upload 'kibana-*'
+buildkite-agent artifact upload "dependencies-$FULL_VERSION.csv"
+buildkite-agent artifact upload "dependencies-$FULL_VERSION.csv.sha512.txt"
+buildkite-agent artifact upload 'i18n/*.json'
 cd -

.buildkite/scripts/steps/artifacts/publish.sh

@@ -51,30 +51,35 @@ trap 'docker logout docker.elastic.co' EXIT
 docker pull docker.elastic.co/infra/release-manager:latest
 
 echo "--- Publish artifacts"
-export VAULT_ROLE_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-role-id)"
-export VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)"
-export VAULT_ADDR="https://secrets.elastic.co:8200"
-docker run --rm \
-  --name release-manager \
-  -e VAULT_ADDR \
-  -e VAULT_ROLE_ID \
-  -e VAULT_SECRET_ID \
-  --mount type=bind,readonly=false,src="$PWD/target",target=/artifacts/target \
-  docker.elastic.co/infra/release-manager:latest \
-    cli collect \
-      --project kibana \
-      --branch "$KIBANA_BASE_BRANCH" \
-      --commit "$GIT_COMMIT" \
-      --workflow "$WORKFLOW" \
-      --version "$BASE_VERSION" \
-      --qualifier "$VERSION_QUALIFIER" \
-      --artifact-set main
-
-ARTIFACTS_SUBDOMAIN="artifacts-$WORKFLOW"
-ARTIFACTS_SUMMARY=$(curl -s "https://$ARTIFACTS_SUBDOMAIN.elastic.co/kibana/latest/$FULL_VERSION.json" | jq -re '.summary_url')
-
-cat << EOF | buildkite-agent annotate --style "info" --context artifacts-summary
+if [[ "$BUILDKITE_BRANCH" == "$KIBANA_BASE_BRANCH" ]]; then
+  export VAULT_ROLE_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-role-id)"
+  export VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)"
+  export VAULT_ADDR="https://secrets.elastic.co:8200"
+  docker run --rm \
+    --name release-manager \
+    -e VAULT_ADDR \
+    -e VAULT_ROLE_ID \
+    -e VAULT_SECRET_ID \
+    --mount type=bind,readonly=false,src="$PWD/target",target=/artifacts/target \
+    docker.elastic.co/infra/release-manager:latest \
+      cli collect \
+        --project kibana \
+        --branch "$KIBANA_BASE_BRANCH" \
+        --commit "$GIT_COMMIT" \
+        --workflow "$WORKFLOW" \
+        --version "$BASE_VERSION" \
+        --qualifier "$VERSION_QUALIFIER" \
+        --artifact-set main
+
+  ARTIFACTS_SUBDOMAIN="artifacts-$WORKFLOW"
+  ARTIFACTS_SUMMARY=$(curl -s "https://$ARTIFACTS_SUBDOMAIN.elastic.co/kibana/latest/$FULL_VERSION.json" | jq -re '.summary_url')
+
+  cat << EOF | buildkite-agent annotate --style "info" --context artifacts-summary
   ### Artifacts Summary
 
   $ARTIFACTS_SUMMARY
 EOF
+
+else
+  echo "Skipping publish for untracked branch $BUILDKITE_BRANCH"
+fi

.buildkite/scripts/steps/package_testing/test.sh

@@ -20,7 +20,7 @@ elif [[ "$TEST_PACKAGE" == "docker" ]]; then
 fi
 cd ..
 
-export VAGRANT_CWD=test/package
+export VAGRANT_CWD=$PWD/test/package
 vagrant up "$TEST_PACKAGE" --no-provision
 
 node scripts/es snapshot \
@@ -29,6 +29,16 @@ node scripts/es snapshot \
   --license=trial &
 while ! timeout 1 bash -c "echo > /dev/tcp/localhost/9200"; do sleep 30; done
 
+function echoKibanaLogs {
+  echo '--- Kibana logs'
+  if [[ "$TEST_PACKAGE" == "deb" ]] || [[ "$TEST_PACKAGE" == "rpm" ]]; then
+    vagrant ssh $TEST_PACKAGE -t -c 'sudo cat /var/log/kibana/kibana.log'
+  elif [[ "$TEST_PACKAGE" == "docker" ]]; then
+    vagrant ssh $TEST_PACKAGE -t -c 'sudo docker logs kibana'
+  fi
+}
+trap "echoKibanaLogs" EXIT
+
 vagrant provision "$TEST_PACKAGE"
 
 export TEST_BROWSER_HEADLESS=1

.ci/ci_groups.yml

@@ -39,4 +39,5 @@ xpack:
   - ciGroup25
   - ciGroup26
   - ciGroup27
-  - ciGroupDocker
+  - ciGroup28
+  - ciGroup29