Merge branch 'main' into use-authorisation-if-present

.backportrc.json

@@ -3,6 +3,7 @@
   "repoName": "kibana",
   "targetBranchChoices": [
     "main",
+    "8.5",
     "8.4",
     "8.3",
     "8.2",
@@ -41,7 +42,7 @@
     "backport"
   ],
   "branchLabelMapping": {
-    "^v8.5.0$": "main",
+    "^v8.6.0$": "main",
     "^v(\\d+).(\\d+).\\d+$": "$1.$2"
   },
   "autoMerge": true,

.buildkite/disabled_jest_configs.json

@@ -0,0 +1,3 @@
+[
+  "x-pack/plugins/watcher/jest.config.js"
+]

.buildkite/ftr_configs.yml

@@ -45,19 +45,19 @@ disabled:
   - x-pack/plugins/observability/e2e/synthetics_run.ts
 
   # Configs that exist but weren't running in CI when this file was introduced
-  - test/visual_regression/config.ts
-  - x-pack/test/visual_regression/config.ts
   - x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/config.ts
   - x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/telemetry/config.ts
   - x-pack/test/alerting_api_integration/spaces_only_legacy/config.ts
-  - x-pack/test/banners_functional/config.ts
   - x-pack/test/cloud_integration/config.ts
   - x-pack/test/performance/config.playwright.ts
   - x-pack/test/load/config.ts
   - x-pack/test/plugin_api_perf/config.js
   - x-pack/test/screenshot_creation/config.ts
   - x-pack/test/fleet_packages/config.ts
 
+  # Scalability testing config that we run in its own pipeline
+  - x-pack/test/scalability/config.ts
+
 defaultQueue: 'n2-4-spot'
 enabled:
   - test/accessibility/config.ts
@@ -117,6 +117,7 @@ enabled:
   - x-pack/test/alerting_api_integration/basic/config.ts
   - x-pack/test/alerting_api_integration/security_and_spaces/group1/config.ts
   - x-pack/test/alerting_api_integration/security_and_spaces/group2/config.ts
+  - x-pack/test/alerting_api_integration/security_and_spaces/group2/config_non_dedicated_task_runner.ts
   - x-pack/test/alerting_api_integration/spaces_only/config.ts
   - x-pack/test/api_integration_basic/config.ts
   - x-pack/test/api_integration/config_security_basic.ts
@@ -125,6 +126,7 @@ enabled:
   - x-pack/test/apm_api_integration/basic/config.ts
   - x-pack/test/apm_api_integration/rules/config.ts
   - x-pack/test/apm_api_integration/trial/config.ts
+  - x-pack/test/banners_functional/config.ts
   - x-pack/test/cases_api_integration/security_and_spaces/config_basic.ts
   - x-pack/test/cases_api_integration/security_and_spaces/config_trial.ts
   - x-pack/test/cases_api_integration/spaces_only/config.ts
@@ -138,6 +140,7 @@ enabled:
   - x-pack/test/detection_engine_api_integration/security_and_spaces/group7/config.ts
   - x-pack/test/detection_engine_api_integration/security_and_spaces/group8/config.ts
   - x-pack/test/detection_engine_api_integration/security_and_spaces/group9/config.ts
+  - x-pack/test/detection_engine_api_integration/security_and_spaces/group10/config.ts
   - x-pack/test/encrypted_saved_objects_api_integration/config.ts
   - x-pack/test/endpoint_api_integration_no_ingest/config.ts
   - x-pack/test/examples/config.ts
@@ -179,7 +182,9 @@ enabled:
   - x-pack/test/functional/apps/maps/group2/config.ts
   - x-pack/test/functional/apps/maps/group3/config.ts
   - x-pack/test/functional/apps/maps/group4/config.ts
-  - x-pack/test/functional/apps/ml/anomaly_detection/config.ts
+  - x-pack/test/functional/apps/ml/anomaly_detection_jobs/config.ts
+  - x-pack/test/functional/apps/ml/anomaly_detection_integrations/config.ts
+  - x-pack/test/functional/apps/ml/anomaly_detection_result_views/config.ts
   - x-pack/test/functional/apps/ml/data_frame_analytics/config.ts
   - x-pack/test/functional/apps/ml/data_visualizer/config.ts
   - x-pack/test/functional/apps/ml/permissions/config.ts
@@ -262,9 +267,10 @@ enabled:
   - x-pack/test/ui_capabilities/spaces_only/config.ts
   - x-pack/test/upgrade_assistant_integration/config.js
   - x-pack/test/usage_collection/config.ts
-  - x-pack/test/performance/journeys/ecommerce_dashboard/config.ts
-  - x-pack/test/performance/journeys/flight_dashboard/config.ts
-  - x-pack/test/performance/journeys/login/config.ts
-  - x-pack/test/performance/journeys/many_fields_discover/config.ts
-  - x-pack/test/performance/journeys/promotion_tracking_dashboard/config.ts
-  - x-pack/test/performance/journeys/web_logs_dashboard/config.ts
+  - x-pack/performance/journeys/ecommerce_dashboard.ts
+  - x-pack/performance/journeys/flight_dashboard.ts
+  - x-pack/performance/journeys/login.ts
+  - x-pack/performance/journeys/many_fields_discover.ts
+  - x-pack/performance/journeys/promotion_tracking_dashboard.ts
+  - x-pack/performance/journeys/web_logs_dashboard.ts
+  - x-pack/performance/journeys/data_stress_test_lens.ts

.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts

@@ -15,6 +15,8 @@ import { load as loadYaml } from 'js-yaml';
 import { BuildkiteClient, BuildkiteStep } from '../buildkite';
 import { CiStatsClient, TestGroupRunOrderResponse } from './client';
 
+import DISABLED_JEST_CONFIGS from '../../disabled_jest_configs.json';
+
 type RunGroup = TestGroupRunOrderResponse['types'][0];
 
 const getRequiredEnv = (name: string) => {
@@ -220,6 +222,7 @@ export async function pickTestGroupRunOrder() {
     ? globby.sync(['**/jest.config.js', '!**/__fixtures__/**'], {
         cwd: process.cwd(),
         absolute: false,
+        ignore: DISABLED_JEST_CONFIGS,
       })
     : [];
 

.buildkite/pipelines/artifacts.yml

@@ -73,13 +73,24 @@ steps:
 
   - command: .buildkite/scripts/steps/artifacts/cloud.sh
     label: 'Cloud Deployment'
-    soft_fail: true
+    soft_fail:
+      - exit_status: 255
+      - exit_status: -1
     agents:
       queue: n2-2
     timeout_in_minutes: 30
     if: "build.env('RELEASE_BUILD') == null || build.env('RELEASE_BUILD') == '' || build.env('RELEASE_BUILD') == 'false'"
     retry:
       automatic:
+        # Timeout and graceful shutdown | ecctl deployment create falure
+        - exit_status: 255
+          limit: 0
+
+        # Timeout and forced shutdown
+        - exit_status: '-1'
+          limit: 0
+
+        # Test failures
         - exit_status: '*'
           limit: 1
 

.buildkite/pipelines/on_merge.yml

@@ -135,7 +135,7 @@ steps:
     label: 'Populate local dev bazel cache (Linux)'
     agents:
       queue: n2-4-spot
-    timeout_in_minutes: 15
+    timeout_in_minutes: 30
     retry:
       automatic:
         - exit_status: '-1'

.buildkite/pipelines/performance/daily.yml

@@ -1,31 +1,39 @@
 steps:
-  - label: ':male-mechanic::skin-tone-2: Pre-Build'
+  - label: '👨‍🔧 Pre-Build'
     command: .buildkite/scripts/lifecycle/pre_build.sh
     agents:
       queue: kibana-default
 
   - wait
 
-  - label: ':factory_worker: Build Kibana Distribution and Plugins'
+  - label: '🧑‍🏭 Build Kibana Distribution and Plugins'
     command: .buildkite/scripts/steps/build_kibana.sh
     agents:
       queue: c2-16
     key: build
+    if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
 
-  - label: ':muscle: Performance Tests with Playwright config'
+  - label: '💪 Performance Tests with Playwright config'
     command: .buildkite/scripts/steps/functional/performance_playwright.sh
     agents:
       queue: kb-static-ubuntu
     depends_on: build
     key: tests
+    timeout_in_minutes: 60
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1
 
-  - label: ':shipit: Performance Tests dataset extraction for scalability benchmarking'
+  - label: '🚢 Performance Tests dataset extraction for scalability benchmarking'
     command: .buildkite/scripts/steps/functional/scalability_dataset_extraction.sh
     agents:
       queue: n2-2
     depends_on: tests
 
-  - label: ':chart_with_upwards_trend: Report performance metrics to ci-stats'
+  - label: '📈 Report performance metrics to ci-stats'
     command: .buildkite/scripts/steps/functional/report_performance_metrics.sh
     agents:
       queue: n2-2
@@ -34,7 +42,7 @@ steps:
   - wait: ~
     continue_on_failure: true
 
-  - label: ':male_superhero::skin-tone-2: Post-Build'
+  - label: '🦸 Post-Build'
     command: .buildkite/scripts/lifecycle/post_build.sh
     agents:
       queue: kibana-default

.buildkite/pipelines/scalability/daily.yml

@@ -0,0 +1,23 @@
+steps:
+  - label: ':male-mechanic::skin-tone-2: Pre-Build'
+    command: .buildkite/scripts/lifecycle/pre_build.sh
+    agents:
+      queue: kibana-default
+    timeout_in_minutes: 10
+
+  - wait
+
+  - label: ':kibana: Scalability Tests'
+    command: .buildkite/scripts/steps/scalability/benchmarking.sh
+    agents:
+      queue: kb-static-scalability
+    timeout_in_minutes: 90
+
+  - wait: ~
+    continue_on_failure: true
+
+  - label: ':male_superhero::skin-tone-2: Post-Build'
+    command: .buildkite/scripts/lifecycle/post_build.sh
+    agents:
+      queue: kibana-default
+    timeout_in_minutes: 10

.buildkite/pull_requests.json

@@ -35,11 +35,19 @@
         "^src/dev/prs/kibana_qa_pr_list\\.json$",
         "^\\.buildkite/pull_requests\\.json$"
       ],
-      "always_require_ci_on_changed": ["^docs/developer/plugin-list.asciidoc$"],
+      "always_require_ci_on_changed": [
+        "^docs/developer/plugin-list.asciidoc$",
+        "/plugins/[^/]+/readme\\.(md|asciidoc)$"
+      ],
       "kibana_versions_check": true,
       "kibana_build_reuse": true,
       "kibana_build_reuse_pipeline_slugs": ["kibana-pull-request", "kibana-on-merge"],
-      "kibana_build_reuse_regexes": ["^test/", "^x-pack/test/"]
+      "kibana_build_reuse_regexes": [
+        "^test/",
+        "^x-pack/test/",
+        "/__snapshots__/",
+        "\\.test\\.(ts|tsx|js|jsx)"
+      ]
     }
   ]
 }

.buildkite/scripts/build_kibana.sh

@@ -10,6 +10,7 @@ echo "--- Build Kibana Distribution"
 
 BUILD_ARGS=""
 is_pr_with_label "ci:build-all-platforms" && BUILD_ARGS="--all-platforms"
+is_pr_with_label "ci:build-example-plugins" && BUILD_ARGS="$BUILD_ARGS --example-plugins"
 is_pr_with_label "ci:build-docker-cross-compile" && BUILD_ARG="$BUILD_ARGS --docker-cross-compile"
 is_pr_with_label "ci:build-os-packages" || BUILD_ARGS="$BUILD_ARGS --skip-os-packages"
 is_pr_with_label "ci:build-canvas-shareable-runtime" || BUILD_ARGS="$BUILD_ARGS --skip-canvas-shareable-runtime"

.buildkite/scripts/common/util.sh

@@ -39,6 +39,7 @@ check_for_changed_files() {
   C_RESET='\033[0m' # Reset color
 
   SHOULD_AUTO_COMMIT_CHANGES="${2:-}"
+  CUSTOM_FIX_MESSAGE="${3:-}"
   GIT_CHANGES="$(git ls-files --modified -- . ':!:.bazelrc')"
 
   if [ "$GIT_CHANGES" ]; then
@@ -75,7 +76,11 @@ check_for_changed_files() {
     else
       echo -e "\n${RED}ERROR: '$1' caused changes to the following files:${C_RESET}\n"
       echo -e "$GIT_CHANGES\n"
-      echo -e "\n${YELLOW}TO FIX: Run '$1' locally, commit the changes and push to your branch${C_RESET}\n"
+      if [ "$CUSTOM_FIX_MESSAGE" ]; then
+        echo "$CUSTOM_FIX_MESSAGE"
+      else
+        echo -e "\n${YELLOW}TO FIX: Run '$1' locally, commit the changes and push to your branch${C_RESET}\n"
+      fi
       exit 1
     fi
   fi

.buildkite/scripts/download_build_artifacts.sh

@@ -15,6 +15,13 @@ if [[ ! -d "$KIBANA_BUILD_LOCATION/bin" ]]; then
   mkdir -p "$KIBANA_BUILD_LOCATION"
   tar -xzf kibana-default.tar.gz -C "$KIBANA_BUILD_LOCATION" --strip=1
 
+  if is_pr_with_label "ci:build-example-plugins"; then
+    # Testing against an example plugin distribution is not supported,
+    # mostly due to snapshot failures when testing UI element lists
+    rm -rf "$KIBANA_BUILD_LOCATION/plugins"
+    mkdir "$KIBANA_BUILD_LOCATION/plugins"
+  fi
+
   cd "$KIBANA_DIR"
 
   tar -xzf ../kibana-default-plugins.tar.gz

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -90,7 +90,7 @@ const uploadPipeline = (pipelineContent: string | object) => {
     }
 
     if (
-      (await doAnyChangesMatch([/^x-pack\/plugins\/apm/])) ||
+      (await doAnyChangesMatch([/^x-pack\/plugins\/apm/, /^packages\/kbn-apm-synthtrace/])) ||
       GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
     ) {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/apm_cypress.yml'));

.buildkite/scripts/steps/artifacts/cloud.sh

@@ -7,33 +7,28 @@ set -euo pipefail
 source "$(dirname "$0")/../../common/util.sh"
 source .buildkite/scripts/steps/artifacts/env.sh
 
-echo "--- Build and publish Cloud image"
+echo "--- Push docker image"
 mkdir -p target
 
-download_artifact "kibana-$FULL_VERSION-linux-x86_64.tar.gz" ./target --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
-
-node scripts/build \
-  --skip-initialize \
-  --skip-generic-folders \
-  --skip-platform-folders \
-  --skip-archives \
-  --docker-images \
-  --skip-docker-ubi \
-  --skip-docker-ubuntu \
-  --skip-docker-contexts
-
-docker load --input target/kibana-cloud-$FULL_VERSION-docker-image.tar.gz
+download_artifact "kibana-cloud-$FULL_VERSION-docker-image.tar.gz" ./target --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+docker load < "target/kibana-cloud-$FULL_VERSION-docker-image.tar.gz"
 
 TAG="$FULL_VERSION-$GIT_COMMIT"
 KIBANA_BASE_IMAGE="docker.elastic.co/kibana-ci/kibana-cloud:$FULL_VERSION"
 KIBANA_TEST_IMAGE="docker.elastic.co/kibana-ci/kibana-cloud:$TAG"
 
+# docker.elastic.co/kibana-ci/kibana-cloud:$FULL_VERSION -> :$FULL_VERSION-$GIT_COMMIT
 docker tag "$KIBANA_BASE_IMAGE" "$KIBANA_TEST_IMAGE"
 
 echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
 trap 'docker logout docker.elastic.co' EXIT
 
-docker push "$KIBANA_TEST_IMAGE"
+if  docker manifest inspect $KIBANA_TEST_IMAGE &> /dev/null; then
+  echo "Cloud image already exists, skipping docker push"
+else
+  docker image push "$KIBANA_TEST_IMAGE"
+fi
+
 docker logout docker.elastic.co
 
 echo "--- Create deployment"
@@ -62,6 +57,7 @@ function shutdown {
 trap "shutdown" EXIT
 
 ecctl deployment create --track --output json --file "$DEPLOYMENT_SPEC" > "$LOGS"
+
 CLOUD_DEPLOYMENT_USERNAME=$(jq -r --slurp '.[]|select(.resources).resources[] | select(.credentials).credentials.username' "$LOGS")
 CLOUD_DEPLOYMENT_PASSWORD=$(jq -r --slurp '.[]|select(.resources).resources[] | select(.credentials).credentials.password' "$LOGS")
 CLOUD_DEPLOYMENT_ID=$(jq -r --slurp '.[0].id' "$LOGS")

.buildkite/scripts/steps/artifacts/env.sh

@@ -15,14 +15,11 @@ fi
 
 if [[ "$RELEASE_BUILD" == "true" ]]; then
   FULL_VERSION="$QUALIFIER_VERSION"
-
-  # Beats artifacts will need to match a specific commit sha that matches other stack images
-  # for release builds.  For now we are skipping Cloud builds until there's a pointer.
-  BUILD_ARGS="--release --skip-docker-cloud --version-qualifier=$VERSION_QUALIFIER"
+  BUILD_ARGS="--release --version-qualifier=$VERSION_QUALIFIER"
   WORKFLOW="staging"
 else
   FULL_VERSION="$QUALIFIER_VERSION-SNAPSHOT"
-  BUILD_ARGS="--skip-docker-cloud --version-qualifier=$VERSION_QUALIFIER"
+  BUILD_ARGS="--version-qualifier=$VERSION_QUALIFIER"
   WORKFLOW="snapshot"
 fi
 

.buildkite/scripts/steps/artifacts/publish.sh

@@ -18,6 +18,8 @@ cd target
 
 download "kibana-$FULL_VERSION-docker-image.tar.gz"
 download "kibana-$FULL_VERSION-docker-image-aarch64.tar.gz"
+download "kibana-cloud-$FULL_VERSION-docker-image.tar.gz"
+download "kibana-cloud-$FULL_VERSION-docker-image-aarch64.tar.gz"
 download "kibana-ubi8-$FULL_VERSION-docker-image.tar.gz"
 
 download "kibana-$FULL_VERSION-arm64.deb"

.buildkite/scripts/steps/checks.sh

@@ -8,6 +8,7 @@ export DISABLE_BOOTSTRAP_VALIDATION=false
 .buildkite/scripts/steps/checks/precommit_hook.sh
 .buildkite/scripts/steps/checks/ftr_configs.sh
 .buildkite/scripts/steps/checks/bazel_packages.sh
+.buildkite/scripts/steps/checks/event_log.sh
 .buildkite/scripts/steps/checks/telemetry.sh
 .buildkite/scripts/steps/checks/ts_projects.sh
 .buildkite/scripts/steps/checks/jest_configs.sh

.buildkite/scripts/steps/checks/bazel_packages.sh

@@ -6,5 +6,12 @@ source .buildkite/scripts/common/util.sh
 
 echo --- Check Bazel Packages Manifest
 node scripts/generate packages_build_manifest
-
 check_for_changed_files 'node scripts/generate packages_build_manifest' true
+
+echo --- Check Codeowners Manifest
+if [ -f ".github/CODEOWNERS" ]; then
+  node scripts/generate codeowners
+  check_for_changed_files 'node scripts/generate codeowners' true
+else
+  echo "skipping, no existing .github/CODEOWNERS file found"
+fi