Merge branch 'main' into run-kibana-quality-gate-suites

.buildkite/ftr_configs.yml

@@ -90,10 +90,13 @@ disabled:
   - x-pack/test_serverless/functional/test_suites/common/config.ts
   - x-pack/test_serverless/functional/test_suites/observability/config.ts
   - x-pack/test_serverless/functional/test_suites/observability/config.feature_flags.ts
+  - x-pack/test_serverless/functional/test_suites/observability/config.examples.ts
   - x-pack/test_serverless/functional/test_suites/search/config.ts
   - x-pack/test_serverless/functional/test_suites/search/config.feature_flags.ts
+  - x-pack/test_serverless/functional/test_suites/search/config.examples.ts
   - x-pack/test_serverless/functional/test_suites/security/config.ts
   - x-pack/test_serverless/functional/test_suites/security/config.feature_flags.ts
+  - x-pack/test_serverless/functional/test_suites/security/config.examples.ts
 
 defaultQueue: 'n2-4-spot'
 enabled:
@@ -279,7 +282,6 @@ enabled:
   - x-pack/test/functional/apps/data_views/config.ts
   - x-pack/test/functional/apps/dev_tools/config.ts
   - x-pack/test/functional/apps/discover/config.ts
-  - x-pack/test/functional/apps/discover_log_explorer/config.ts
   - x-pack/test/functional/apps/graph/config.ts
   - x-pack/test/functional/apps/grok_debugger/config.ts
   - x-pack/test/functional/apps/home/config.ts
@@ -313,6 +315,7 @@ enabled:
   - x-pack/test/functional/apps/ml/short_tests/config.ts
   - x-pack/test/functional/apps/ml/stack_management_jobs/config.ts
   - x-pack/test/functional/apps/monitoring/config.ts
+  - x-pack/test/functional/apps/observability_log_explorer/config.ts
   - x-pack/test/functional/apps/remote_clusters/config.ts
   - x-pack/test/functional/apps/reporting_management/config.ts
   - x-pack/test/functional/apps/rollup_job/config.ts
@@ -419,6 +422,7 @@ enabled:
   - x-pack/performance/journeys/ecommerce_dashboard_saved_search_only.ts
   - x-pack/performance/journeys/ecommerce_dashboard_tsvb_gauge_only.ts
   - x-pack/performance/journeys/dashboard_listing_page.ts
+  - x-pack/performance/journeys/tags_listing_page.ts
   - x-pack/performance/journeys/cloud_security_dashboard.ts
   - x-pack/performance/journeys/apm_service_inventory.ts
   - x-pack/test/custom_branding/config.ts

.buildkite/pipelines/on_merge_unsupported_ftrs.yml

@@ -55,7 +55,7 @@ steps:
       queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 120
-    parallelism: 6
+    parallelism: 14
     retry:
       automatic:
         - exit_status: '*'

.buildkite/pipelines/pipeline.kibana-serverless-release.yaml

@@ -0,0 +1,9 @@
+steps:
+  - label: ":releasethekraken: Release kibana"
+    # https://regex101.com/r/tY52jo/1
+    if: build.tag =~ /^deploy@\d+\$/
+    trigger: gpctl-promote
+    build:
+      env:
+        SERVICE_COMMIT_HASH: "${BUILDKITE_COMMIT:0:12}"
+        REMOTE_SERVICE_CONFIG: https://raw.githubusercontent.com/elastic/serverless-gitops/main/gen/gpctl/kibana/config.yaml

.buildkite/pipelines/pull_request/base.yml

@@ -69,6 +69,19 @@ steps:
         - exit_status: '-1'
           limit: 3
 
+  - command: SERVERLESS_ENVIRONMENT=observability.examples .buildkite/scripts/steps/functional/serverless_ftr.sh
+    label: 'Serverless Observability Examples Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 40
+    soft_fail:
+      - exit_status: 10
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+
   - command: SERVERLESS_ENVIRONMENT=search .buildkite/scripts/steps/functional/serverless_ftr.sh
     label: 'Serverless Search Tests'
     agents:
@@ -82,6 +95,19 @@ steps:
         - exit_status: '-1'
           limit: 3
 
+  - command: SERVERLESS_ENVIRONMENT=search.examples .buildkite/scripts/steps/functional/serverless_ftr.sh
+    label: 'Serverless Search Examples Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 40
+    soft_fail:
+      - exit_status: 10
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+
   - command: SERVERLESS_ENVIRONMENT=security .buildkite/scripts/steps/functional/serverless_ftr.sh
     label: 'Serverless Security Tests'
     agents:
@@ -95,27 +121,26 @@ steps:
         - exit_status: '-1'
           limit: 3
 
-  - command: .buildkite/scripts/steps/functional/security_serverless.sh
-    label: 'Serverless Security Cypress Tests'
+  - command: SERVERLESS_ENVIRONMENT=security.examples .buildkite/scripts/steps/functional/serverless_ftr.sh
+    label: 'Serverless Security Examples Tests'
     agents:
       queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 40
-    parallelism: 16
-    soft_fail: true
+    soft_fail:
+      - exit_status: 10
     retry:
       automatic:
-        - exit_status: '*'
-          limit: 1
-    artifact_paths:
-      - "target/kibana-security-solution/**/*"
+        - exit_status: '-1'
+          limit: 3
 
-  - command: .buildkite/scripts/steps/functional/security_serverless_defend_workflows.sh
-    label: 'Serverless Security Defend Workflows Cypress Tests'
+  - command: .buildkite/scripts/steps/functional/security_serverless.sh
+    label: 'Serverless Security Cypress Tests'
     agents:
       queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 40
+    parallelism: 16
     soft_fail: true
     retry:
       automatic:
@@ -124,6 +149,21 @@ steps:
     artifact_paths:
       - "target/kibana-security-solution/**/*"
 
+  # status_exception: Native role management is not enabled in this Elasticsearch instance
+  # - command: .buildkite/scripts/steps/functional/security_serverless_defend_workflows.sh
+  #   label: 'Serverless Security Defend Workflows Cypress Tests'
+  #   agents:
+  #     queue: n2-4-spot
+  #   depends_on: build
+  #   timeout_in_minutes: 40
+  #   soft_fail: true
+  #   retry:
+  #     automatic:
+  #       - exit_status: '*'
+  #         limit: 1
+  #   artifact_paths:
+  #     - "target/kibana-security-solution/**/*"
+
   - command: .buildkite/scripts/steps/functional/security_serverless_investigations.sh
     label: 'Serverless Security Investigations Cypress Tests'
     agents:

.buildkite/pipelines/pull_request/defend_workflows.yml

@@ -4,7 +4,7 @@ steps:
     agents:
       queue: n2-4-spot
     depends_on: build
-    timeout_in_minutes: 120
+    timeout_in_minutes: 60
     parallelism: 2
     retry:
       automatic:
@@ -16,9 +16,9 @@ steps:
   - command: .buildkite/scripts/steps/functional/defend_workflows_vagrant.sh
     label: 'Defend Workflows Endpoint Cypress Tests'
     agents:
-      queue: n2-16-virt
+      queue: n2-4-virt
     depends_on: build
-    timeout_in_minutes: 120
+    timeout_in_minutes: 60
     parallelism: 6
     retry:
       automatic:

.buildkite/pipelines/pull_request/osquery_cypress.yml

@@ -20,19 +20,22 @@ steps:
     depends_on: build
     timeout_in_minutes: 50
     soft_fail: true
-    artifact_paths:
-      - "target/kibana-osquery/**/*"
-
-  - command: .buildkite/scripts/steps/functional/security_serverless_osquery.sh
-    label: 'Serverless Osquery Cypress Tests'
-    agents:
-      queue: n2-4-spot
-    depends_on: build
-    timeout_in_minutes: 50
-    parallelism: 6
     retry:
-      automatic:
-        - exit_status: '*'
-          limit: 1
+      automatic: false
     artifact_paths:
       - "target/kibana-osquery/**/*"
+
+  # Error: self-signed certificate in certificate chain
+  # - command: .buildkite/scripts/steps/functional/security_serverless_osquery.sh
+  #   label: 'Serverless Osquery Cypress Tests'
+  #   agents:
+  #     queue: n2-4-spot
+  #   depends_on: build
+  #   timeout_in_minutes: 50
+  #   parallelism: 6
+  #   retry:
+  #     automatic:
+  #       - exit_status: '*'
+  #         limit: 1
+  #   artifact_paths:
+  #     - "target/kibana-osquery/**/*"

.buildkite/pipelines/pull_request/security_solution.yml

@@ -11,21 +11,21 @@ steps:
         - exit_status: '*'
           limit: 1
     artifact_paths:
-      - "target/kibana-security-solution/**/*"
+      - 'target/kibana-security-solution/**/*'
 
   - command: .buildkite/scripts/steps/functional/security_solution_explore.sh
     label: 'Explore - Security Solution Cypress Tests'
     agents:
       queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 60
-    parallelism: 2
+    parallelism: 4
     retry:
       automatic:
         - exit_status: '*'
           limit: 1
     artifact_paths:
-      - "target/kibana-security-solution/**/*"
+      - 'target/kibana-security-solution/**/*'
 
   - command: .buildkite/scripts/steps/functional/security_solution_investigations.sh
     label: 'Investigations - Security Solution Cypress Tests'
@@ -39,7 +39,7 @@ steps:
         - exit_status: '*'
           limit: 1
     artifact_paths:
-      - "target/kibana-security-solution/**/*"
+      - 'target/kibana-security-solution/**/*'
 
   - command: .buildkite/scripts/steps/functional/security_solution_burn.sh
     label: 'Security Solution Cypress tests, burning changed specs'
@@ -48,6 +48,15 @@ steps:
     depends_on: build
     timeout_in_minutes: 120
     parallelism: 1
+    retry:
+      automatic: false
     soft_fail: true
     artifact_paths:
-      - "target/kibana-security-solution/**/*"
+      - 'target/kibana-security-solution/**/*'
+
+  - command: .buildkite/scripts/steps/code_generation/security_solution_codegen.sh
+    label: 'Security Solution OpenAPI codegen'
+    agents:
+      queue: n2-2-spot
+    timeout_in_minutes: 60
+    parallelism: 1

.buildkite/pipelines/serverless.yml

@@ -34,6 +34,19 @@ steps:
         - exit_status: '*'
           limit: 1
 
+  - command: SERVERLESS_ENVIRONMENT=observability.examples .buildkite/scripts/steps/functional/serverless_ftr.sh
+    label: 'Serverless Observability Examples Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 40
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1
+
   - command: SERVERLESS_ENVIRONMENT=search .buildkite/scripts/steps/functional/serverless_ftr.sh
     label: 'Serverless Search Tests'
     agents:
@@ -47,6 +60,19 @@ steps:
         - exit_status: '*'
           limit: 1
 
+  - command: SERVERLESS_ENVIRONMENT=search.examples .buildkite/scripts/steps/functional/serverless_ftr.sh
+    label: 'Serverless Search Examples Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 40
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1
+
   - command: SERVERLESS_ENVIRONMENT=security .buildkite/scripts/steps/functional/serverless_ftr.sh
     label: 'Serverless Security Tests'
     agents:
@@ -60,6 +86,19 @@ steps:
         - exit_status: '*'
           limit: 1
 
+  - command: SERVERLESS_ENVIRONMENT=security.examples .buildkite/scripts/steps/functional/serverless_ftr.sh
+    label: 'Serverless Security Examples Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 40
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+        - exit_status: '*'
+          limit: 1
+
   - command: .buildkite/scripts/steps/functional/security_serverless.sh
     label: 'Serverless Security Cypress Tests'
     agents:
@@ -75,7 +114,7 @@ steps:
       - "target/kibana-security-serverless/**/*"
 
   - command: .buildkite/scripts/steps/functional/security_serverless_explore.sh
-    label: 'Explore - Security Solution Cypress Tests'
+    label: 'Serverless Explore - Security Solution Cypress Tests'
     agents:
       queue: n2-4-spot
     depends_on: build
@@ -89,7 +128,7 @@ steps:
       - "target/kibana-security-serverless/**/*"
 
   - command: .buildkite/scripts/steps/functional/security_serverless_investigations.sh
-    label: 'Investigations - Security Solution Cypress Tests'
+    label: 'Serverless Investigations - Security Solution Cypress Tests'
     agents:
       queue: n2-4-spot
     depends_on: build

.buildkite/scripts/common/util.sh

@@ -31,7 +31,7 @@ check_for_changed_files() {
 
   SHOULD_AUTO_COMMIT_CHANGES="${2:-}"
   CUSTOM_FIX_MESSAGE="${3:-}"
-  GIT_CHANGES="$(git ls-files --modified -- . ':!:.bazelrc')"
+  GIT_CHANGES="$(git status --porcelain -- . ':!:.bazelrc')"
 
   if [ "$GIT_CHANGES" ]; then
     if ! is_auto_commit_disabled && [[ "$SHOULD_AUTO_COMMIT_CHANGES" == "true" && "${BUILDKITE_PULL_REQUEST:-}" ]]; then
@@ -54,7 +54,7 @@ check_for_changed_files() {
       git config --global user.name kibanamachine
       git config --global user.email '[email protected]'
       gh pr checkout "${BUILDKITE_PULL_REQUEST}"
-      git add -u -- . ':!.bazelrc'
+      git add -A -- . ':!.bazelrc'
 
       git commit -m "$NEW_COMMIT_MESSAGE"
       git push

.buildkite/scripts/lifecycle/pre_command.sh

@@ -2,6 +2,11 @@
 
 set -euo pipefail
 
+if [[ "$BUILDKITE_COMMAND" =~ ^"buildkite-agent pipeline upload" ]]; then
+  echo "Skipped pre-command when running the Upload pipeline"
+  exit 0
+fi
+
 source .buildkite/scripts/common/util.sh
 
 echo '--- Setup environment vars'

.buildkite/scripts/steps/artifacts/docker_image.sh

@@ -98,12 +98,14 @@ steps:
   - label: ":argo: Update kibana image tag for kibana-controller using gpctl"
     async: true
     branches: main
-    trigger: gpctl-promote
+    trigger: gpctl-promote-with-e2e-tests
     build:
       env:
         SERVICE_COMMIT_HASH: "$GIT_ABBREV_COMMIT"
-        REMOTE_SERVICE_CONFIG: https://raw.githubusercontent.com/elastic/serverless-gitops/main/gen/gpctl/kibana/config.yaml
-
+        SERVICE: kibana-controller
+        NAMESPACE: kibana-ci
+        IMAGE_NAME: kibana-serverless
+        REMOTE_SERVICE_CONFIG: https://raw.githubusercontent.com/elastic/serverless-gitops/main/gen/gpctl/kibana/dev.yaml
 EOF
 
 else