Merge branch 'main' into compatible-schema-change-check

* main: (1294 commits)
  [SecuritySolution] Refactor security packages (#155365)
  [Discover] Show "Temporary" badge for ad-hoc data views in Alerts flyout (#155717)
  [RAM] Conditional actions feedback on pr review (#155804)
  [Files] Adds bulk delete method (#155628)
  [Lens] Use proper way to generate absolute short URL (#155512)
  [Guided onboarding] Use Kibana features to grant access (#155065)
  [Index Management] Fix duped mock (#155844)
  [Lens] Enhance visualization modifier popup with layer palette (#155280)
  Fix flaky combobox tests on role management screen (#155711)
  [Infrastructure UI] Create InventoryViewsService and InventoryViewsClient (#155126)
  [Fleet] always create agent upload write indices (#155729)
  [Fleet] [Cloud Security Posture] Add CloudFormation agent install method (#155045)
  Add tech preview label for search applications (#155649)
  [ML] AIOps: Stabilize flaky functional tests. (#155710)
  [ES UI Shared] Migrate JsonEditor to monaco (#155610)
  [Security Solution] Fixes security_solution storybooks always rendering in a flyout (#155814)
  [Synthetics] Make error popover disappear `onMouseLeave` of metric item card (#155800)
  Remove Exploratory View components from Observability (#155629)
  [Discover] Remove redundant "Filter was added" toast (#155645)
  [RAM][Security Solution][Alerts] Support the ability to trigger a rule action per alert generated (#153611) (#155384)
  ...

.buildkite/ftr_configs.yml

@@ -43,16 +43,20 @@ disabled:
   - x-pack/test/threat_intelligence_cypress/visual_config.ts
   - x-pack/test/threat_intelligence_cypress/cli_config_parallel.ts
   - x-pack/test/threat_intelligence_cypress/config.ts
-  - x-pack/test/functional_enterprise_search/with_host_configured.config.ts
+  - x-pack/test/functional_enterprise_search/visual_config.ts
+  - x-pack/test/functional_enterprise_search/cli_config.ts
   - x-pack/plugins/apm/ftr_e2e/ftr_config_open.ts
   - x-pack/plugins/apm/ftr_e2e/ftr_config_run.ts
   - x-pack/plugins/apm/ftr_e2e/ftr_config.ts
+  - x-pack/plugins/profiling/e2e/ftr_config_open.ts
+  - x-pack/plugins/profiling/e2e/ftr_config_runner.ts
+  - x-pack/plugins/profiling/e2e/ftr_config.ts
 
   # Elastic Synthetics configs
   - x-pack/plugins/synthetics/e2e/config.ts
   - x-pack/plugins/synthetics/e2e/synthetics_run.ts
   - x-pack/plugins/ux/e2e/synthetics_run.ts
-  - x-pack/plugins/observability/e2e/synthetics_run.ts
+  - x-pack/plugins/exploratory_view/e2e/synthetics_run.ts
 
   # Configs that exist but weren't running in CI when this file was introduced
   - x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/config.ts
@@ -68,6 +72,9 @@ disabled:
   # Scalability testing config that we run in its own pipeline
   - x-pack/test/scalability/config.ts
 
+  # Asset Manager configs, in tech preview, will move to enabled after more stability introduced
+  - x-pack/test/api_integration/apis/asset_manager/config.ts
+
 defaultQueue: 'n2-4-spot'
 enabled:
   - test/accessibility/config.ts
@@ -88,6 +95,7 @@ enabled:
   - test/functional/apps/discover/embeddable/config.ts
   - test/functional/apps/discover/group1/config.ts
   - test/functional/apps/discover/group2/config.ts
+  - test/functional/apps/discover/group3/config.ts
   - test/functional/apps/getting_started/config.ts
   - test/functional/apps/home/config.ts
   - test/functional/apps/kibana_overview/config.ts
@@ -146,6 +154,7 @@ enabled:
   - x-pack/test/api_integration/config_security_basic.ts
   - x-pack/test/api_integration/config_security_trial.ts
   - x-pack/test/api_integration/apis/aiops/config.ts
+  - x-pack/test/api_integration/apis/asset_manager/config_when_disabled.ts
   - x-pack/test/api_integration/apis/cases/config.ts
   - x-pack/test/api_integration/apis/cloud_security_posture/config.ts
   - x-pack/test/api_integration/apis/console/config.ts
@@ -176,6 +185,7 @@ enabled:
   - x-pack/test/api_integration/apis/uptime/config.ts
   - x-pack/test/api_integration/apis/watcher/config.ts
   - x-pack/test/apm_api_integration/basic/config.ts
+  - x-pack/test/apm_api_integration/cloud/config.ts
   - x-pack/test/apm_api_integration/rules/config.ts
   - x-pack/test/apm_api_integration/trial/config.ts
   - x-pack/test/banners_functional/config.ts
@@ -215,7 +225,6 @@ enabled:
   - x-pack/test/functional_basic/apps/transform/feature_controls/config.ts
   - x-pack/test/functional_cors/config.ts
   - x-pack/test/functional_embedded/config.ts
-  - x-pack/test/functional_enterprise_search/without_host_configured.config.ts
   - x-pack/test/functional_execution_context/config.ts
   - x-pack/test/functional_with_es_ssl/apps/cases/group1/config.ts
   - x-pack/test/functional_with_es_ssl/apps/cases/group2/config.ts

.buildkite/pipelines/artifacts_trigger.yml

@@ -0,0 +1,6 @@
+steps:
+  - command: .buildkite/scripts/steps/artifacts/trigger.sh
+    label: Trigger artifacts build
+    agents:
+      queue: kibana-default
+    timeout_in_minutes: 10

.buildkite/pipelines/on_merge.yml

@@ -78,6 +78,14 @@ steps:
         - exit_status: '*'
           limit: 1
 
+  - command: '.buildkite/scripts/steps/functional/on_merge_unsupported_ftrs.sh'
+    label: Trigger unsupported ftr tests
+    timeout_in_minutes: 10
+    depends_on:
+      - build
+    agents:
+      queue: 'kibana-default'
+
   - command: .buildkite/scripts/steps/lint.sh
     label: 'Linting'
     agents:

.buildkite/pipelines/on_merge_unsupported_ftrs.yml

@@ -0,0 +1,79 @@
+steps:
+  - command: .buildkite/scripts/lifecycle/pre_build.sh
+    label: Pre-Build
+    timeout_in_minutes: 10
+    agents:
+      queue: kibana-default
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/build_kibana.sh
+    label: Build Kibana Distribution and Plugins
+    agents:
+      queue: n2-16-spot
+    key: build
+    if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
+    timeout_in_minutes: 60
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+
+  - command: .buildkite/scripts/steps/functional/apm_cypress.sh
+    label: 'APM Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 120
+    parallelism: 4
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/profiling_cypress.sh
+    label: 'Profling Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 120
+    parallelism: 4
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/security_solution.sh
+    label: 'Security Solution Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 120
+    parallelism: 4
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/synthetics_plugin.sh
+    label: 'Synthetics @elastic/synthetics Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 120
+    artifact_paths:
+      - 'x-pack/plugins/synthetics/e2e/.journeys/**/*'
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1

.buildkite/pipelines/pull_request/exploratory_view_plugin.yml

@@ -0,0 +1,15 @@
+steps:
+  - command: .buildkite/scripts/steps/functional/exploratory_view_plugin.sh
+    label: 'Exploratory View @elastic/synthetics Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 120
+    artifact_paths:
+      - 'x-pack/plugins/exploratory_view/e2e/.journeys/**/*'
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1

.buildkite/pipelines/pull_request/profiling_cypress.yml

@@ -0,0 +1,14 @@
+steps:
+  - command: .buildkite/scripts/steps/functional/profiling_cypress.sh
+    label: 'Profiling Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 120
+    parallelism: 2
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1

.buildkite/scripts/build_kibana.sh

@@ -30,6 +30,7 @@ if is_pr_with_label "ci:build-cloud-image"; then
   --docker-push \
   --skip-docker-ubi \
   --skip-docker-ubuntu \
+  --skip-docker-serverless \
   --skip-docker-contexts
   docker logout docker.elastic.co
 

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -115,6 +115,13 @@ const uploadPipeline = (pipelineContent: string | object) => {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/apm_cypress.yml'));
     }
 
+    if (
+      (await doAnyChangesMatch([/^x-pack\/plugins\/profiling/])) ||
+      GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
+    ) {
+      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/profiling_cypress.yml'));
+    }
+
     if (
       (await doAnyChangesMatch([/^x-pack\/plugins\/fleet/, /^x-pack\/test\/fleet_cypress/])) ||
       GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
@@ -124,31 +131,27 @@ const uploadPipeline = (pipelineContent: string | object) => {
     }
 
     if (
-      (await doAnyChangesMatch([/^x-pack\/plugins\/osquery/, /^x-pack\/test\/osquery_cypress/])) ||
-      GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
+      ((await doAnyChangesMatch([/^x-pack\/plugins\/osquery/, /^x-pack\/test\/osquery_cypress/])) ||
+        GITHUB_PR_LABELS.includes('ci:all-cypress-suites')) &&
+      !GITHUB_PR_LABELS.includes('ci:skip-cypress-osquery')
     ) {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/osquery_cypress.yml'));
     }
 
-    if (await doAnyChangesMatch([/^x-pack\/plugins\/observability/])) {
-      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/observability_plugin.yml'));
+    if (await doAnyChangesMatch([/^x-pack\/plugins\/exploratory_view/])) {
+      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/exploratory_view_plugin.yml'));
     }
 
     if (
       await doAnyChangesMatch([
         /^x-pack\/plugins\/synthetics/,
-        /^x-pack\/plugins\/observability\/public\/components\/shared\/exploratory_view/,
+        /^x-pack\/plugins\/exploratory_view/,
       ])
     ) {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/synthetics_plugin.yml'));
     }
 
-    if (
-      await doAnyChangesMatch([
-        /^x-pack\/plugins\/ux/,
-        /^x-pack\/plugins\/observability\/public\/components\/shared\/exploratory_view/,
-      ])
-    ) {
+    if (await doAnyChangesMatch([/^x-pack\/plugins\/ux/, /^x-pack\/plugins\/exploratory_view/])) {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/ux_plugin_e2e.yml'));
     }
 

.buildkite/scripts/steps/artifacts/docker_image.sh

@@ -7,13 +7,14 @@ set -euo pipefail
 source .buildkite/scripts/steps/artifacts/env.sh
 
 GIT_ABBREV_COMMIT=${BUILDKITE_COMMIT:0:12}
-KIBANA_IMAGE="docker.elastic.co/kibana-ci/kibana:git-$GIT_ABBREV_COMMIT"
+KIBANA_IMAGE_INPUT="docker.elastic.co/kibana-ci/kibana-serverless:git-$GIT_ABBREV_COMMIT"
+KIBANA_IMAGE_OUTPUT="docker.elastic.co/kibana-ci/kibana:git-$GIT_ABBREV_COMMIT"
 
 echo "--- Verify manifest does not already exist"
 echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
 trap 'docker logout docker.elastic.co' EXIT
 
-if docker manifest inspect $KIBANA_IMAGE &> /dev/null; then
+if docker manifest inspect $KIBANA_IMAGE_OUTPUT &> /dev/null; then
   echo "Manifest already exists, exiting"
   exit 1
 fi
@@ -26,42 +27,42 @@ node scripts/build \
   --docker-images \
   --docker-namespace="kibana-ci" \
   --docker-tag="git-$GIT_ABBREV_COMMIT" \
+  --skip-docker-ubuntu \
   --skip-docker-ubi \
   --skip-docker-cloud \
   --skip-docker-contexts
 
 echo "--- Tag images"
-docker rmi "$KIBANA_IMAGE"
-docker load < "target/kibana-$BASE_VERSION-docker-image.tar.gz"
-docker tag "$KIBANA_IMAGE" "$KIBANA_IMAGE-amd64"
+docker rmi "$KIBANA_IMAGE_INPUT"
+docker load < "target/kibana-serverless-$BASE_VERSION-docker-image.tar.gz"
+docker tag "$KIBANA_IMAGE_INPUT" "$KIBANA_IMAGE_OUTPUT-amd64"
 
-docker rmi "$KIBANA_IMAGE"
-docker load < "target/kibana-$BASE_VERSION-docker-image-aarch64.tar.gz"
-docker tag "$KIBANA_IMAGE" "$KIBANA_IMAGE-arm64"
+docker rmi "$KIBANA_IMAGE_INPUT"
+docker load < "target/kibana-serverless-$BASE_VERSION-docker-image-aarch64.tar.gz"
+docker tag "$KIBANA_IMAGE_INPUT" "$KIBANA_IMAGE_OUTPUT-arm64"
 
 echo "--- Push images"
-docker image push "$KIBANA_IMAGE-arm64"
-docker image push "$KIBANA_IMAGE-amd64"
+docker image push "$KIBANA_IMAGE_OUTPUT-arm64"
+docker image push "$KIBANA_IMAGE_OUTPUT-amd64"
 
 echo "--- Create manifest"
-docker rmi "$KIBANA_IMAGE"
 docker manifest create \
-  "$KIBANA_IMAGE" \
-  --amend "$KIBANA_IMAGE-arm64" \
-  --amend "$KIBANA_IMAGE-amd64"
+  "$KIBANA_IMAGE_OUTPUT" \
+  --amend "$KIBANA_IMAGE_OUTPUT-arm64" \
+  --amend "$KIBANA_IMAGE_OUTPUT-amd64"
 
 echo "--- Push manifest"
-docker manifest push "$KIBANA_IMAGE"
+docker manifest push "$KIBANA_IMAGE_OUTPUT"
 docker logout docker.elastic.co
 
 cat << EOF | buildkite-agent annotate --style "info" --context image
   ### Container Images
 
-  Manifest: \`$KIBANA_IMAGE\`
+  Manifest: \`$KIBANA_IMAGE_OUTPUT\`
 
-  AMD64: \`$KIBANA_IMAGE-amd64\`
+  AMD64: \`$KIBANA_IMAGE_OUTPUT-amd64\`
 
-  ARM64: \`$KIBANA_IMAGE-arm64\`
+  ARM64: \`$KIBANA_IMAGE_OUTPUT-arm64\`
 EOF
 
 echo "--- Build dependencies report"
@@ -76,18 +77,19 @@ buildkite-agent artifact upload "kibana-$BASE_VERSION-docker-image-aarch64.tar.g
 buildkite-agent artifact upload "dependencies-$GIT_ABBREV_COMMIT.csv"
 cd -
 
+# This part is related with updating the configuration of kibana-controller,
+# so that new stack instances contain the latest and greatest image of kibana,
+# and the respective stack components of course.
 echo "--- Trigger image tag update"
 if [[ "$BUILDKITE_BRANCH" == "$KIBANA_BASE_BRANCH" ]]; then
-
   cat << EOF | buildkite-agent pipeline upload
 steps:
-  - trigger: k8s-gitops-update-image-tag
+  - trigger: serverless-gitops-update-stack-image-tag
+    async: true
     label: ":argo: Update image tag for Kibana"
     branches: main
     build:
       env:
-        MODE: sed
-        TARGET_FILE: kibana-controller.yaml
         IMAGE_TAG: "git-$GIT_ABBREV_COMMIT"
         SERVICE: kibana-controller
         NAMESPACE: kibana-ci

.buildkite/scripts/steps/artifacts/env.sh

@@ -23,9 +23,16 @@ else
   WORKFLOW="snapshot"
 fi
 
+ARTIFACTS_SUBDOMAIN="artifacts-$WORKFLOW"
+ARTIFACTS_MANIFEST_FQDN="https://$ARTIFACTS_SUBDOMAIN.elastic.co"
+KIBANA_MANIFEST_LATEST="$ARTIFACTS_MANIFEST_FQDN/kibana/latest/$FULL_VERSION.json"
+BEATS_MANIFEST_LATEST="$ARTIFACTS_MANIFEST_FQDN/beats/latest/$FULL_VERSION.json"
+
 export VERSION_QUALIFIER
 export BASE_VERSION
 export QUALIFIER_VERSION
 export FULL_VERSION
 export BUILD_ARGS
 export WORKFLOW
+export KIBANA_MANIFEST_LATEST
+export BEATS_MANIFEST_LATEST