Merge branch 'main' into feature/synthtrace-timeseries

.bazelrc

@@ -2,14 +2,6 @@
 # Import shared settings first so we can override below
 import %workspace%/.bazelrc.common
 
-# Remote cache settings for local env
-# build --remote_cache=grpcs://cloud.buildbuddy.io
-# build --incompatible_remote_results_ignore_disk=true
-# build --noremote_upload_local_results
-# build --remote_timeout=30
-# build --remote_header=x-buildbuddy-api-key=3EYk49W2NefOx2n3yMze
-# build --remote_accept_cached=true
-
 # Enable this in case you want to share your build info
 # build --build_metadata=VISIBILITY=PUBLIC
 build --build_metadata=TEST_GROUPS=//packages

.buildkite/pipelines/artifacts.yml

@@ -4,29 +4,68 @@ steps:
     agents:
       queue: c2-16
     timeout_in_minutes: 60
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
 
   - wait
 
   - command: TEST_PACKAGE=deb .buildkite/scripts/steps/package_testing/test.sh
-    label: Package testing for deb
+    label: Artifact Testing
     agents:
       queue: n2-4-virt
-    timeout_in_minutes: 20
+    timeout_in_minutes: 30
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
 
   - command: TEST_PACKAGE=rpm .buildkite/scripts/steps/package_testing/test.sh
-    label: Package testing for rpm
+    label: Artifact Testing
     agents:
       queue: n2-4-virt
-    timeout_in_minutes: 20
+    timeout_in_minutes: 30
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
 
   - command: TEST_PACKAGE=docker .buildkite/scripts/steps/package_testing/test.sh
-    label: Package testing for docker
+    label: Artifact Testing
     agents:
       queue: n2-4-virt
-    timeout_in_minutes: 20
+    timeout_in_minutes: 30
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
 
   - command: .buildkite/scripts/steps/artifacts/docker_context.sh
-    label: 'Docker Build Context'
+    label: 'Docker Context Verification'
     agents:
       queue: n2-2
-    timeout_in_minutes: 20
+    timeout_in_minutes: 30
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/artifacts/cloud.sh
+    label: 'Cloud Deployment'
+    agents:
+      queue: n2-2
+    timeout_in_minutes: 30
+    if: "build.env('RELEASE_BUILD') == null || build.env('RELEASE_BUILD') == '' || build.env('RELEASE_BUILD') == 'false'"
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - wait
+
+  - command: .buildkite/scripts/steps/artifacts/publish.sh
+    label: 'Publish Kibana Artifacts'
+    agents:
+      queue: n2-2
+    timeout_in_minutes: 30

.buildkite/pipelines/es_snapshots/verify.yml

@@ -29,7 +29,7 @@ steps:
 
   - command: .buildkite/scripts/steps/functional/xpack_cigroup.sh
     label: 'Default CI Group'
-    parallelism: 27
+    parallelism: 31
     agents:
       queue: n2-4
     depends_on: build
@@ -40,18 +40,6 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - command: CI_GROUP=Docker .buildkite/scripts/steps/functional/xpack_cigroup.sh
-    label: 'Docker CI Group'
-    agents:
-      queue: n2-4
-    depends_on: build
-    timeout_in_minutes: 120
-    key: default-cigroup-docker
-    retry:
-      automatic:
-        - exit_status: '*'
-          limit: 1
-
   - command: .buildkite/scripts/steps/functional/oss_cigroup.sh
     label: 'OSS CI Group'
     parallelism: 11
@@ -91,7 +79,6 @@ steps:
       queue: kibana-default
     depends_on:
       - default-cigroup
-      - default-cigroup-docker
       - oss-cigroup
       - jest-integration
       - api-integration

.buildkite/pipelines/flaky_tests/groups.json

@@ -32,11 +32,7 @@
     {
       "key": "xpack/cigroup",
       "name": "Default CI Group",
-      "ciGroups": 27
-    },
-    {
-      "key": "xpack/cigroup/Docker",
-      "name": "Default CI Group Docker"
+      "ciGroups": 30
     },
     {
       "key": "xpack/firefox",
@@ -47,4 +43,4 @@
       "name": "Default Accessibility"
     }
   ]
-}
+}

.buildkite/pipelines/on_merge.yml

@@ -51,7 +51,7 @@ steps:
 
   - command: .buildkite/scripts/steps/functional/xpack_cigroup.sh
     label: 'Default CI Group'
-    parallelism: 27
+    parallelism: 31
     agents:
       queue: n2-4-spot
     depends_on: build
@@ -64,20 +64,6 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - command: CI_GROUP=Docker .buildkite/scripts/steps/functional/xpack_cigroup.sh
-    label: 'Docker CI Group'
-    agents:
-      queue: n2-4-spot
-    depends_on: build
-    timeout_in_minutes: 120
-    key: default-cigroup-docker
-    retry:
-      automatic:
-        - exit_status: '-1'
-          limit: 3
-        - exit_status: '*'
-          limit: 1
-
   - command: .buildkite/scripts/steps/functional/oss_cigroup.sh
     label: 'OSS CI Group'
     parallelism: 11

.buildkite/pipelines/pull_request/base.yml

@@ -17,26 +17,16 @@ steps:
 
   - command: .buildkite/scripts/steps/functional/xpack_cigroup.sh
     label: 'Default CI Group'
-    parallelism: 27
+    parallelism: 31
     agents:
-      queue: n2-4
+      queue: n2-4-spot-2
     depends_on: build
     timeout_in_minutes: 150
     key: default-cigroup
     retry:
       automatic:
-        - exit_status: '*'
-          limit: 1
-
-  - command: CI_GROUP=Docker .buildkite/scripts/steps/functional/xpack_cigroup.sh
-    label: 'Docker CI Group'
-    agents:
-      queue: n2-4
-    depends_on: build
-    timeout_in_minutes: 120
-    key: default-cigroup-docker
-    retry:
-      automatic:
+        - exit_status: '-1'
+          limit: 3
         - exit_status: '*'
           limit: 1
 

.buildkite/scripts/build_kibana_plugins.sh

@@ -3,18 +3,4 @@
 set -euo pipefail
 
 echo "--- Build Platform Plugins"
-node scripts/build_kibana_platform_plugins \
-  --scan-dir "$KIBANA_DIR/test/analytics/__fixtures__/plugins" \
-  --scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \
-  --scan-dir "$KIBANA_DIR/test/interpreter_functional/plugins" \
-  --scan-dir "$KIBANA_DIR/test/common/fixtures/plugins" \
-  --scan-dir "$KIBANA_DIR/examples" \
-  --scan-dir "$XPACK_DIR/test/plugin_functional/plugins" \
-  --scan-dir "$XPACK_DIR/test/functional_with_es_ssl/fixtures/plugins" \
-  --scan-dir "$XPACK_DIR/test/alerting_api_integration/plugins" \
-  --scan-dir "$XPACK_DIR/test/plugin_api_integration/plugins" \
-  --scan-dir "$XPACK_DIR/test/plugin_api_perf/plugins" \
-  --scan-dir "$XPACK_DIR/test/licensing_plugin/plugins" \
-  --scan-dir "$XPACK_DIR/test/usage_collection/plugins" \
-  --scan-dir "$XPACK_DIR/test/security_functional/fixtures/common" \
-  --scan-dir "$XPACK_DIR/examples"
+node scripts/build_kibana_platform_plugins --examples --test-plugins

.buildkite/scripts/common/setup_bazel.sh

@@ -15,8 +15,8 @@ if [[ "${BAZEL_CACHE_MODE:-none}" == read* ]]; then
   echo "[bazel] enabling caching"
 cat <<EOF >> $KIBANA_DIR/.bazelrc
   build --bes_results_url=https://app.buildbuddy.io/invocation/
-  build --bes_backend=grpcs://cloud.buildbuddy.io
-  build --remote_cache=grpcs://cloud.buildbuddy.io
+  build --bes_backend=grpcs://remote.buildbuddy.io
+  build --remote_cache=grpcs://remote.buildbuddy.io
   build --remote_timeout=3600
   build --remote_header=x-buildbuddy-api-key=$KIBANA_BUILDBUDDY_CI_API_KEY
 EOF

.buildkite/scripts/lifecycle/pre_build.sh

@@ -12,3 +12,11 @@ export CI_STATS_TOKEN="$(retry 5 5 vault read -field=api_token secret/kibana-iss
 export CI_STATS_HOST="$(retry 5 5 vault read -field=api_host secret/kibana-issues/dev/kibana_ci_stats)"
 
 node "$(dirname "${0}")/ci_stats_start.js"
+
+# We resolve the latest manifest URL at the beginning of the build to ensure that all steps in the build will use the same manifest
+# Otherwise, the manifest could change if a step is running around the time that a new one is promoted
+if [[ ! "${ES_SNAPSHOT_MANIFEST:-}" ]]; then
+  BUCKET=$(curl -s "https://storage.googleapis.com/kibana-ci-es-snapshots-daily/$(cat package.json | jq -r .version)/manifest-latest-verified.json" | jq -r .bucket)
+  ES_SNAPSHOT_MANIFEST_DEFAULT="https://storage.googleapis.com/$BUCKET/manifest.json"
+  buildkite-agent meta-data set ES_SNAPSHOT_MANIFEST_DEFAULT "$ES_SNAPSHOT_MANIFEST_DEFAULT"
+fi

.buildkite/scripts/lifecycle/pre_command.sh

@@ -65,6 +65,15 @@ EOF
   fi
 }
 
+# If a custom manifest isn't specified, then use the default one that we resolve earlier in the build
+{
+  if [[ ! "${ES_SNAPSHOT_MANIFEST:-}" ]]; then
+    ES_SNAPSHOT_MANIFEST=${ES_SNAPSHOT_MANIFEST:-$(buildkite-agent meta-data get ES_SNAPSHOT_MANIFEST_DEFAULT --default '')}
+    export ES_SNAPSHOT_MANIFEST
+    echo "Using default ES Snapshot Manifest: $ES_SNAPSHOT_MANIFEST"
+  fi
+}
+
 # Setup CI Stats
 {
   CI_STATS_BUILD_ID="$(buildkite-agent meta-data get ci_stats_build_id --default '')"

.buildkite/scripts/steps/artifacts/build.sh

@@ -4,44 +4,23 @@ set -euo pipefail
 
 .buildkite/scripts/bootstrap.sh
 
-if [[ "${RELEASE_BUILD:-}" == "true" ]]; then
-  VERSION="$(jq -r '.version' package.json)"
-  RELEASE_ARG="--release"
-else
-  VERSION="$(jq -r '.version' package.json)-SNAPSHOT"
-  RELEASE_ARG=""
-fi
+source .buildkite/scripts/steps/artifacts/env.sh
 
-echo "--- Build Kibana Distribution"
-node scripts/build "$RELEASE_ARG" --all-platforms --debug --docker-cross-compile --skip-docker-cloud
+echo "--- Build Kibana artifacts"
+node scripts/build --all-platforms --debug --docker-cross-compile $(echo "$BUILD_ARGS")
 
-echo "--- Build dependencies report"
-node scripts/licenses_csv_report "--csv=target/dependencies-$VERSION.csv"
-
-# Release verification
-if [[ "${RELEASE_BUILD:-}" == "true" ]]; then
-  echo "--- Build and push Kibana Cloud Distribution"
-  # This doesn't meet the requirements for a release image, implementation TBD
-  # Beats artifacts will need to match a specific commit sha that matches other stack iamges
-  # For now this is a placeholder step that will allow us to run automated Cloud tests
-  # against a best guess approximation of a release image
-  echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
-  trap 'docker logout docker.elastic.co' EXIT
+echo "--- Extract default i18n messages"
+mkdir -p target/i18n
+node scripts/i18n_extract --output-dir=target/i18n
 
-  node scripts/build \
-    "$RELEASE_ARG" \
-    --skip-initialize \
-    --skip-generic-folders \
-    --skip-platform-folders \
-    --skip-archives \
-    --docker-images \
-    --docker-tag-qualifier="$GIT_COMMIT" \
-    --docker-push \
-    --skip-docker-ubi \
-    --skip-docker-ubuntu \
-    --skip-docker-contexts
-fi
+echo "--- Build dependencies report"
+node scripts/licenses_csv_report "--csv=target/dependencies-$FULL_VERSION.csv"
+(cd target; sha512sum "dependencies-$FULL_VERSION.csv" > "dependencies-$FULL_VERSION.csv.sha512.txt")
 
+echo "--- Upload Kibana Artifacts"
 cd target
-buildkite-agent artifact upload "*"
-cd -
+buildkite-agent artifact upload 'kibana-*'
+buildkite-agent artifact upload "dependencies-$FULL_VERSION.csv"
+buildkite-agent artifact upload "dependencies-$FULL_VERSION.csv.sha512.txt"
+buildkite-agent artifact upload 'i18n/*.json'
+cd -