Merge branch 'main' into dataset-quality-plugin-render-time

.buildkite/ftr_configs.yml

@@ -193,7 +193,6 @@ enabled:
   - x-pack/test/api_integration/config_security_basic.ts
   - x-pack/test/api_integration/config_security_trial.ts
   - x-pack/test/api_integration/apis/aiops/config.ts
-  - x-pack/test/api_integration/apis/asset_manager/config_when_disabled.ts
   - x-pack/test/api_integration/apis/cases/config.ts
   - x-pack/test/api_integration/apis/content_management/config.ts
   - x-pack/test/api_integration/apis/cloud_security_posture/config.ts

.buildkite/pipeline-resource-definitions/kibana-serverless-emergency-release.yml

@@ -0,0 +1,30 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: buildkite-pipeline-kibana-emergency-release
+  description: Emergency release
+  links:
+    - title: Pipeline
+      url: https://buildkite.com/elastic/kibana-emergency-release
+spec:
+  type: buildkite-pipeline
+  owner: group:kibana-operations
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / emergency release
+    spec:
+      repository: elastic/kibana
+      provider_settings:
+        trigger_mode: none
+      pipeline_file: ".buildkite/pipelines/emergency_release.yml"
+      teams:
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-release-operators:
+          access_level: BUILD_AND_READ
+        everyone:
+          access_level: READ_ONLY

.buildkite/pipeline-resource-definitions/kibana-serverless-quality-gates-emergency.yml

@@ -0,0 +1,33 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: kibana-tests-emergency-pipeline
+  description: Definition of the kibana pipeline
+  links:
+    - title: Pipeline
+      url: https://buildkite.com/elastic/kibana-tests-emergency
+spec:
+  type: buildkite-pipeline
+  owner: group:kibana-tech-leads
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana-tests-emergency
+      description: Pipeline that tests the service integration in various environments
+    spec:
+      repository: elastic/kibana
+      pipeline_file: ./.buildkite/pipelines/quality-gates/emergency/pipeline.emergency.kibana-tests.yaml
+      provider_settings:
+        trigger_mode: none
+      teams:
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-release-operators:
+          access_level: BUILD_AND_READ
+        cloud-tooling:
+          access_level: BUILD_AND_READ
+        everyone:
+          access_level: READ_ONLY

.buildkite/pipeline-resource-definitions/kibana-serverless-quality-gates.yml

@@ -0,0 +1,33 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: kibana-tests-pipeline
+  description: Definition of the kibana pipeline
+  links:
+    - title: Pipeline
+      url: https://buildkite.com/elastic/kibana-tests
+spec:
+  type: buildkite-pipeline
+  owner: group:kibana-tech-leads
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana-tests
+      description: Pipeline that tests the service integration in various environments
+    spec:
+      repository: elastic/kibana
+      pipeline_file: ./.buildkite/pipelines/quality-gates/pipeline.kibana-tests.yaml
+      provider_settings:
+        trigger_mode: none
+      teams:
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-release-operators:
+          access_level: BUILD_AND_READ
+        cloud-tooling:
+          access_level: BUILD_AND_READ
+        everyone:
+          access_level: READ_ONLY

.buildkite/pipeline-resource-definitions/locations.yml

@@ -27,6 +27,9 @@ spec:
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-performance-data-set-extraction-daily.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-pr.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-purge-cloud-deployments.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-emergency-release.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-quality-gates.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-quality-gates-emergency.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-release-testing.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-release.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/scalability_testing-daily.yml

.buildkite/pipeline-utils/agent_images.ts

@@ -20,7 +20,7 @@ const DEFAULT_AGENT_IMAGE_CONFIG: AgentImageConfig = {
 const FIPS_AGENT_IMAGE_CONFIG: AgentImageConfig = {
   provider: 'gcp',
   image: 'family/kibana-fips-ubuntu-2004',
-  imageProject: 'elastic-images-qa',
+  imageProject: 'elastic-images-prod',
 };
 
 const GITHUB_PR_LABELS = process.env.GITHUB_PR_LABELS ?? '';

.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml

@@ -5,7 +5,7 @@
 # SKIP_VERIFICATION: if set to 1/true, it will skip running all tests
 # SKIP_CYPRESS: if set to 1/true, it will skip running the cypress tests
 # FTR_EXTRA_ARGS: a string argument, if passed, it will be forwarded verbatim to the FTR run script
-# ES_SERVERLESS_IMAGE: the tag for the docker image to test, in the form of docker.elastic.co/elasticsearch-ci/elasticsearch-serverless:$TAG
+# ES_SERVERLESS_IMAGE: the full image path for the docker image to test
 # BUILDKITE_COMMIT: the commit hash of the kibana branch to test
 
 agents:
@@ -16,16 +16,7 @@ agents:
 
 steps:
   - label: "Annotate runtime parameters"
-    command: |
-      buildkite-agent annotate --context kibana-commit --style info "Kibana build hash: $BUILDKITE_BRANCH / $BUILDKITE_COMMIT"
-      cat << EOF | buildkite-agent annotate --context es-serverless-image --style info
-        ES Serverless image: \`$ES_SERVERLESS_IMAGE\`
-
-        To run this locally:
-        \`\`\`
-        node scripts/es serverless --image $ES_SERVERLESS_IMAGE
-        \`\`\`
-      EOF
+    command: .buildkite/scripts/steps/es_serverless/annotate_runtime_parameters.sh
 
   - group: "(:kibana: x :elastic:) Trigger Kibana Serverless suite"
     if: "build.env('SKIP_VERIFICATION') != '1' && build.env('SKIP_VERIFICATION') != 'true'"

.buildkite/pipelines/fips.yml

@@ -1,32 +1,63 @@
 env:
-  DISABLE_CI_STATS_SHIPPING: "true"
+  DISABLE_CI_STATS_SHIPPING: 'true'
+  KBN_ENABLE_FIPS: 'true'
+  TEST_BROWSER_HEADLESS: 1
+agents:
+  provider: 'gcp'
+  image: 'family/kibana-fips-ubuntu-2004'
+  imageProject: 'elastic-images-prod'
 steps:
+  - command: .buildkite/scripts/lifecycle/pre_build.sh
+    label: Pre-Build
+    key: pre-build
+    timeout_in_minutes: 10
+    agents:
+      machineType: n2-standard-2
+
+  - wait
+
   - command: .buildkite/scripts/steps/build_kibana.sh
     label: Build Kibana Distribution and Plugins
     agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
       machineType: n2-standard-16
       preemptible: true
     key: build
     if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
+    depends_on: pre-build
     timeout_in_minutes: 60
     retry:
       automatic:
-        - exit_status: "-1"
+        - exit_status: '-1'
           limit: 3
 
   - wait
 
-  - command: TEST_PACKAGE=fips .buildkite/scripts/steps/package_testing/test.sh
-    label: "Smoke testing for FIPS"
+  - command: .buildkite/scripts/steps/checks/verify_fips_enabled.sh
+    label: 'Verify FIPS Enabled'
+    depends_on: build
+    timeout_in_minutes: 10
+    agents:
+      machineType: n2-standard-2
+      preemptible: true
+
+  - command: .buildkite/scripts/steps/fips/smoke_test.sh
+    label: 'Pick Smoke Test Group Run Order'
+    depends_on: build
+    timeout_in_minutes: 10
+    env:
+      FTR_CONFIGS_SCRIPT: '.buildkite/scripts/steps/test/ftr_configs.sh'
+      FTR_EXTRA_ARGS: '$FTR_EXTRA_ARGS'
+      LIMIT_CONFIG_TYPE: 'functional'
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - wait: ~
+    continue_on_failure: true
+
+  - command: .buildkite/scripts/lifecycle/post_build.sh
+    label: Post-Build
+    timeout_in_minutes: 10
     agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
-      enableNestedVirtualization: true
-      localSsds: 1
-      localSsdInterface: nvme
-      machineType: n2-standard-4
-    timeout_in_minutes: 600
+      machineType: n2-standard-2

.buildkite/pipelines/pull_request/observability_onboarding_cypress.yml

@@ -8,7 +8,6 @@ steps:
       - build
       - quick_checks
     timeout_in_minutes: 120
-    parallelism: 2
     retry:
       automatic:
         - exit_status: '-1'

.buildkite/pipelines/quality-gates/emergency/pipeline.emergency.kibana-tests.yaml

@@ -0,0 +1,33 @@
+# This pipeline serves as the entry point for your service's quality gates definitions. When
+# properly configured, it will be invoked automatically as part of the automated
+# promotion process once a new version was rolled out in one of the various cloud stages.
+#
+# The updated environment is provided via ENVIRONMENT variable. The seedling
+# step will branch and execute pipeline snippets at the following location:
+# pipeline.tests-qa.yaml
+# pipeline.tests-staging.yaml
+# pipeline.tests-production.yaml
+#
+# Docs: https://docs.elastic.dev/serverless/qualitygates
+
+agents:
+  cpu: 2
+  ephemeralStorage: "20G"
+  memory: "8G"
+
+env:
+  SKIP_NODE_SETUP: true
+  TEAM_CHANNEL: "#kibana-mission-control"
+  ENVIRONMENT: ${ENVIRONMENT?}
+
+steps:
+  - label: ":pipeline::grey_question::seedling: Trigger Kibana Tests for ${ENVIRONMENT}"
+    env:
+      QG_PIPELINE_LOCATION: ".buildkite/pipelines/quality-gates/emergency"
+    command: "make -C /agent run-environment-tests" # will trigger https://buildkite.com/elastic/kibana-tests-emergency
+    agents:
+      image: "docker.elastic.co/ci-agent-images/quality-gate-seedling:0.0.4"
+
+notify:
+  - slack: "${TEAM_CHANNEL?}"
+    if: build.branch == "main" && build.state == "failed"

.buildkite/pipelines/quality-gates/emergency/pipeline.tests-production.yaml

@@ -0,0 +1,37 @@
+# These pipeline steps constitute the quality gate for your service within the production environment.
+# Incorporate any necessary additional logic to validate the service's integrity.
+# A failure in this pipeline build will prevent further progression to the subsequent stage.
+
+steps:
+  - label: ":kibana: SLO check"
+    trigger: "serverless-quality-gates" # https://buildkite.com/elastic/serverless-quality-gates
+    build:
+      message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-production.yaml)"
+      env:
+        TARGET_ENV: production
+        CHECK_SLO: true
+        CHECK_SLO_TAG: kibana
+        CHECK_SLO_WAITING_PERIOD: 15m
+        CHECK_SLO_BURN_RATE_THRESHOLD: 0.1
+    soft_fail: true
+
+  - label: ":rocket: control-plane e2e tests"
+    if: build.env("ENVIRONMENT") == "production-canary"
+    trigger: "ess-k8s-production-e2e-tests" # https://buildkite.com/elastic/ess-k8s-production-e2e-tests
+    build:
+      env:
+        REGION_ID: aws-us-east-1
+        NAME_PREFIX: ci_test_kibana-promotion_
+      message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-production.yaml)"
+
+  - label: ":cookie: 24h bake time before continuing promotion"
+    if: build.env("ENVIRONMENT") == "production-canary"
+    command: "sleep 86400"
+    soft_fail:
+      # A manual cancel of that step produces return code 255.
+      # We're treating this case as a soft fail to allow manual bake time skipping.
+      # To stop the promotion entirely, instead click the "Cancel" button at the top of the page
+      - exit_status: 255
+    agents:
+      # How long can this agent live for in minutes - 25 hours
+      instanceMaxAge: 1500

.buildkite/pipelines/quality-gates/emergency/pipeline.tests-qa.yaml

@@ -0,0 +1,12 @@
+# These pipeline steps constitute the quality gate for your service within the QA environment.
+# Incorporate any necessary additional logic to validate the service's integrity.
+# A failure in this pipeline build will prevent further progression to the subsequent stage.
+
+steps:
+  - label: ":rocket: control-plane e2e tests"
+    trigger: "ess-k8s-qa-e2e-tests-daily" # https://buildkite.com/elastic/ess-k8s-qa-e2e-tests-daily
+    build:
+      env:
+        REGION_ID: aws-eu-west-1
+        NAME_PREFIX: ci_test_kibana-promotion_
+      message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-qa.yaml)"

.buildkite/pipelines/quality-gates/emergency/pipeline.tests-staging.yaml

@@ -0,0 +1,45 @@
+# These pipeline steps constitute the quality gate for your service within the staging environment.
+# Incorporate any necessary additional logic to validate the service's integrity.
+# A failure in this pipeline build will prevent further progression to the subsequent stage.
+
+steps:
+  - label: ":rocket: control-plane e2e tests"
+    trigger: "ess-k8s-staging-e2e-tests" # https://buildkite.com/elastic/ess-k8s-staging-e2e-tests
+    build:
+      env:
+        REGION_ID: aws-us-east-1
+        NAME_PREFIX: ci_test_kibana-promotion_
+      message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-staging.yaml)"
+
+  - label: ":kibana: Kibana Serverless Tests for ${ENVIRONMENT}"
+    trigger: appex-qa-serverless-kibana-ftr-tests # https://buildkite.com/elastic/appex-qa-serverless-kibana-ftr-tests
+    soft_fail: true # Remove when tests stabilize
+    build:
+      env:
+        ENVIRONMENT: ${ENVIRONMENT}
+        EC_ENV: staging
+        EC_REGION: aws-us-east-1
+        RETRY_TESTS_ON_FAIL: "true"
+      message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-staging.yaml)"
+
+  - label: ":rocket: Fleet synthetic monitor to check the long standing project"
+    trigger: "serverless-quality-gates"
+    build:
+      message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-staging.yaml)"
+      env:
+        TARGET_ENV: staging
+        CHECK_SYNTHETICS: true
+        CHECK_SYNTHETICS_TAG: "fleet"
+        CHECK_SYNTHETICS_MINIMUM_RUNS: 3
+        MAX_FAILURES: 2
+        CHECK_SYNTHETIC_MAX_POLL: 50
+    soft_fail: true
+
+  - wait: ~
+
+  - group: "Kibana Release Manager"
+    steps:
+      - label: ":judge::seedling: Trigger Manual Tests Phase"
+        command: "make -C /agent trigger-manual-verification-phase"
+        agents:
+          image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.6"

.buildkite/pipelines/quality-gates/pipeline.tests-staging.yaml

@@ -21,7 +21,7 @@ steps:
         EC_REGION: aws-us-east-1
         RETRY_TESTS_ON_FAIL: "true"
       message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-staging.yaml)"
-  
+
   - label: ":rocket: Fleet synthetic monitor to check the long standing project"
     trigger: "serverless-quality-gates"
     build:

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -83,7 +83,7 @@ const getPipeline = (filename: string, removeSteps = true) => {
 
     if (
       (await doAnyChangesMatch([
-        /^x-pack\/plugins\/observability_onboarding/,
+        /^x-pack\/plugins\/observability_solution\/observability_onboarding/,
         /^x-pack\/plugins\/fleet/,
       ])) ||
       GITHUB_PR_LABELS.includes('ci:all-cypress-suites')