Merge remote-tracking branch 'origin/main' into 3454-dataset-quality-…

…enable-pages-for-metrics-and-traces

.buildkite/ftr_base_serverless_configs.yml

@@ -1,6 +1,8 @@
 disabled:
   # Base config files, only necessary to inform config finding script
 
+  # Serverless deployment-agnostic default config for api-integration tests
+  - x-pack/test/api_integration/deployment_agnostic/default_configs/serverless.config.base.ts
   # Serverless base config files
   - x-pack/test_serverless/api_integration/config.base.ts
   - x-pack/test_serverless/functional/config.base.ts

.buildkite/ftr_oblt_serverless_configs.yml

@@ -26,3 +26,5 @@ enabled:
   - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group5.ts
   - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group6.ts
   - x-pack/test_serverless/functional/test_suites/observability/config.screenshots.ts
+  # serverless config files that run deployment-agnostic tests
+  - x-pack/test/api_integration/deployment_agnostic/oblt.serverless.config.ts

.buildkite/ftr_oblt_stateful_configs.yml

@@ -25,6 +25,7 @@ defaultQueue: 'n2-4-spot'
 enabled:
   - x-pack/test/alerting_api_integration/observability/config.ts
   - x-pack/test/api_integration/apis/logs_ui/config.ts
+  - x-pack/test/api_integration/apis/logs_shared/config.ts
   - x-pack/test/api_integration/apis/metrics_ui/config.ts
   - x-pack/test/api_integration/apis/osquery/config.ts
   - x-pack/test/api_integration/apis/synthetics/config.ts

.buildkite/ftr_platform_stateful_configs.yml

@@ -1,4 +1,6 @@
 disabled:
+  # Stateful base config for deployment-agnostic tests
+  - x-pack/test/api_integration/deployment_agnostic/default_configs/stateful.config.base.ts
   # Base config files, only necessary to inform config finding script
   - test/functional/config.base.js
   - test/functional/firefox/config.base.ts
@@ -155,7 +157,6 @@ enabled:
   - x-pack/test/api_integration/apis/monitoring/config.ts
   - x-pack/test/api_integration/apis/monitoring_collection/config.ts
   - x-pack/test/api_integration/apis/osquery/config.ts
-  - x-pack/test/api_integration/apis/painless_lab/config.ts
   - x-pack/test/api_integration/apis/search/config.ts
   - x-pack/test/api_integration/apis/searchprofiler/config.ts
   - x-pack/test/api_integration/apis/security/config.ts
@@ -359,3 +360,5 @@ enabled:
   - x-pack/performance/journeys_e2e/apm_service_inventory.ts
   - x-pack/performance/journeys_e2e/infra_hosts_view.ts
   - x-pack/test/custom_branding/config.ts
+  # stateful config files that run deployment-agnostic tests
+  - x-pack/test/api_integration/deployment_agnostic/stateful.config.ts

.buildkite/ftr_search_serverless_configs.yml

@@ -16,3 +16,5 @@ enabled:
   - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group4.ts
   - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group5.ts
   - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group6.ts
+  # serverless config files that run deployment-agnostic tests
+  - x-pack/test/api_integration/deployment_agnostic/search.serverless.config.ts

.buildkite/ftr_security_serverless_configs.yml

@@ -97,3 +97,5 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/response_actions/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_endpoint/configs/serverless.endpoint.config.ts
   - x-pack/test/security_solution_endpoint/configs/serverless.integrations.config.ts
+  # serverless config files that run deployment-agnostic tests
+  - x-pack/test/api_integration/deployment_agnostic/security.serverless.config.ts

.buildkite/ftr_security_stateful_configs.yml

@@ -82,4 +82,6 @@ enabled:
   - x-pack/test/security_solution_endpoint/configs/integrations.config.ts
   - x-pack/test/api_integration/apis/cloud_security_posture/config.ts
   - x-pack/test/cloud_security_posture_api/config.ts
-  - x-pack/test/cloud_security_posture_functional/config.ts
+  - x-pack/test/cloud_security_posture_functional/config.ts
+  - x-pack/test/cloud_security_posture_functional/config.agentless.ts
+  - x-pack/test/cloud_security_posture_functional/data_views/config.ts

.buildkite/pipeline-resource-definitions/kibana-fips-daily.yml

@@ -19,12 +19,12 @@ spec:
       description: Run Kibana FIPS smoke tests
     spec:
       env:
-        SLACK_NOTIFICATIONS_CHANNEL: "#kibana-fips-ftr-errors"
-        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "true"
+        SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
       repository: elastic/kibana
       branch_configuration: main
       default_branch: main
-      pipeline_file: ".buildkite/pipelines/fips.yml"
+      pipeline_file: '.buildkite/pipelines/fips.yml'
       provider_settings:
         trigger_mode: none
       schedules:

.buildkite/pipeline-resource-definitions/scripts/validate-pipeline-definition.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 
+# renovate.json tracks this file for updating the pipelib docker image, update the path if you move this file
 # This script is used to validate a single RRE for a pipeline definition.
 
 TARGET_FILE=$1
@@ -16,7 +17,7 @@ FOLDER_NAME=$(dirname "$ABSOLUTE_PATH")
 
 docker run -it \
  --mount type=bind,source="$FOLDER_NAME",target=/home/app/ \
-  docker.elastic.co/ci-agent-images/pipelib \
+  docker.elastic.co/ci-agent-images/pipelib:0.8.0@sha256:641d7fc6cfe473900a1fbe49876762916d804b09fdf2945f74e9f803f3073779 \
   rre validate --backstage-entity-aware "/home/app/$FILE_NAME"
 
 if [ $? -ne 0 ]; then

.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml

@@ -44,26 +44,26 @@ steps:
         timeout_in_minutes: 60
         retry:
           automatic:
-            - exit_status: '-1'
+            - exit_status: "-1"
               limit: 3
 
       - label: "Pick Test Group Run Order (FTR + Integration)"
         command: .buildkite/scripts/steps/test/pick_test_group_run_order.sh
         depends_on: build
         timeout_in_minutes: 10
         env:
-          FTR_CONFIGS_SCRIPT: 'TEST_ES_SERVERLESS_IMAGE=$ES_SERVERLESS_IMAGE .buildkite/scripts/steps/test/ftr_configs.sh'
-          JEST_INTEGRATION_SCRIPT: 'TEST_ES_SERVERLESS_IMAGE=$ES_SERVERLESS_IMAGE .buildkite/scripts/steps/test/jest_integration.sh'
-          FTR_CONFIG_PATTERNS: '**/test_serverless/**'
-          FTR_EXTRA_ARGS: '$FTR_EXTRA_ARGS'
-          LIMIT_CONFIG_TYPE: 'functional,integration'
+          FTR_CONFIGS_SCRIPT: "TEST_ES_SERVERLESS_IMAGE=$ES_SERVERLESS_IMAGE .buildkite/scripts/steps/test/ftr_configs.sh"
+          JEST_INTEGRATION_SCRIPT: "TEST_ES_SERVERLESS_IMAGE=$ES_SERVERLESS_IMAGE .buildkite/scripts/steps/test/jest_integration.sh"
+          FTR_CONFIG_PATTERNS: "**/test_serverless/**,**/test/security_solution_api_integration/**/serverless.config.ts"
+          FTR_EXTRA_ARGS: "$FTR_EXTRA_ARGS"
+          LIMIT_CONFIG_TYPE: "functional,integration"
         retry:
           automatic:
-            - exit_status: '*'
+            - exit_status: "*"
               limit: 1
 
       - command: .buildkite/scripts/steps/functional/security_serverless_entity_analytics.sh
-        label: 'Serverless Entity Analytics - Security Solution Cypress Tests'
+        label: "Serverless Entity Analytics - Security Solution Cypress Tests"
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           image: family/kibana-ubuntu-2004
@@ -76,11 +76,11 @@ steps:
         parallelism: 3
         retry:
           automatic:
-            - exit_status: '-1'
+            - exit_status: "-1"
               limit: 1
 
       - command: .buildkite/scripts/steps/functional/security_serverless_explore.sh
-        label: 'Serverless Explore - Security Solution Cypress Tests'
+        label: "Serverless Explore - Security Solution Cypress Tests"
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           image: family/kibana-ubuntu-2004
@@ -93,11 +93,11 @@ steps:
         parallelism: 4
         retry:
           automatic:
-            - exit_status: '-1'
+            - exit_status: "-1"
               limit: 1
 
       - command: .buildkite/scripts/steps/functional/security_serverless_investigations.sh
-        label: 'Serverless Investigations - Security Solution Cypress Tests'
+        label: "Serverless Investigations - Security Solution Cypress Tests"
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           image: family/kibana-ubuntu-2004
@@ -110,11 +110,11 @@ steps:
         parallelism: 8
         retry:
           automatic:
-            - exit_status: '-1'
+            - exit_status: "-1"
               limit: 1
 
       - command: .buildkite/scripts/steps/functional/security_serverless_rule_management.sh
-        label: 'Serverless Rule Management - Security Solution Cypress Tests'
+        label: "Serverless Rule Management - Security Solution Cypress Tests"
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           image: family/kibana-ubuntu-2004
@@ -127,11 +127,11 @@ steps:
         parallelism: 5
         retry:
           automatic:
-            - exit_status: '-1'
+            - exit_status: "-1"
               limit: 1
 
       - command: .buildkite/scripts/steps/functional/security_serverless_rule_management_prebuilt_rules.sh
-        label: 'Serverless Rule Management - Prebuilt Rules - Security Solution Cypress Tests'
+        label: "Serverless Rule Management - Prebuilt Rules - Security Solution Cypress Tests"
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           image: family/kibana-ubuntu-2004
@@ -144,11 +144,11 @@ steps:
         parallelism: 1
         retry:
           automatic:
-            - exit_status: '-1'
+            - exit_status: "-1"
               limit: 1
 
       - command: .buildkite/scripts/steps/functional/security_serverless_detection_engine.sh
-        label: 'Serverless Detection Engine - Security Solution Cypress Tests'
+        label: "Serverless Detection Engine - Security Solution Cypress Tests"
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           image: family/kibana-ubuntu-2004
@@ -161,11 +161,11 @@ steps:
         parallelism: 5
         retry:
           automatic:
-            - exit_status: '-1'
+            - exit_status: "-1"
               limit: 1
 
       - command: .buildkite/scripts/steps/functional/security_serverless_detection_engine_exceptions.sh
-        label: 'Serverless Detection Engine - Exceptions - Security Solution Cypress Tests'
+        label: "Serverless Detection Engine - Exceptions - Security Solution Cypress Tests"
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           image: family/kibana-ubuntu-2004
@@ -178,11 +178,11 @@ steps:
         parallelism: 2
         retry:
           automatic:
-            - exit_status: '-1'
+            - exit_status: "-1"
               limit: 1
 
       - command: .buildkite/scripts/steps/functional/security_serverless_ai_assistant.sh
-        label: 'Serverless AI Assistant - Security Solution Cypress Tests'
+        label: "Serverless AI Assistant - Security Solution Cypress Tests"
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           image: family/kibana-ubuntu-2004
@@ -195,11 +195,11 @@ steps:
         parallelism: 1
         retry:
           automatic:
-            - exit_status: '-1'
+            - exit_status: "-1"
               limit: 1
 
       - command: .buildkite/scripts/steps/functional/defend_workflows_serverless.sh
-        label: 'Defend Workflows Cypress Tests on Serverless'
+        label: "Defend Workflows Cypress Tests on Serverless"
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           image: family/kibana-ubuntu-2004
@@ -214,11 +214,11 @@ steps:
         parallelism: 12
         retry:
           automatic:
-            - exit_status: '-1'
+            - exit_status: "-1"
               limit: 1
 
       - command: .buildkite/scripts/steps/functional/security_serverless_osquery.sh
-        label: 'Serverless Osquery Cypress Tests'
+        label: "Serverless Osquery Cypress Tests"
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           image: family/kibana-ubuntu-2004
@@ -231,7 +231,7 @@ steps:
         parallelism: 7
         retry:
           automatic:
-            - exit_status: '-1'
+            - exit_status: "-1"
               limit: 1
 
   - wait: ~
@@ -241,6 +241,6 @@ steps:
 
   - wait: ~
 
-  - label: 'Post-Build'
+  - label: "Post-Build"
     command: .buildkite/scripts/lifecycle/post_build.sh
     timeout_in_minutes: 10

.buildkite/pipelines/sonarqube.yml

@@ -2,8 +2,8 @@ env:
   SKIP_NODE_SETUP: true
 
 steps:
-  - label: ":sonarqube: Continuous Code Inspection"
+  - label: ':sonarqube: Continuous Code Inspection'
     agents:
       image: docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest
-      memory: 16G
+      memory: 32G
     command: /scan-source-code.sh

.buildkite/scripts/common/env.sh

@@ -23,12 +23,14 @@ if [[ -d /opt/local-ssd/buildkite ]]; then
   mkdir -p "$TMPDIR"
 fi
 
-KIBANA_PKG_BRANCH="$(jq -r .branch "$KIBANA_DIR/package.json")"
-export KIBANA_PKG_BRANCH
-export KIBANA_BASE_BRANCH="$KIBANA_PKG_BRANCH"
+if command -v jq >/dev/null 2>&1; then
+  KIBANA_PKG_BRANCH="$(jq -r .branch "$KIBANA_DIR/package.json")"
+  export KIBANA_PKG_BRANCH
+  export KIBANA_BASE_BRANCH="$KIBANA_PKG_BRANCH"
 
-KIBANA_PKG_VERSION="$(jq -r .version "$KIBANA_DIR/package.json")"
-export KIBANA_PKG_VERSION
+  KIBANA_PKG_VERSION="$(jq -r .version "$KIBANA_DIR/package.json")"
+  export KIBANA_PKG_VERSION
+fi
 
 # Detects and exports the final target branch when using a merge queue
 if [[ "${BUILDKITE_BRANCH:-}" == "gh-readonly-queue"* ]]; then

.buildkite/scripts/common/util.sh

@@ -36,7 +36,7 @@ check_for_changed_files() {
   GIT_CHANGES="$(git status --porcelain -- . ':!:.bazelrc' ':!:config/node.options' ':!config/kibana.yml')"
 
   if [ "$GIT_CHANGES" ]; then
-    if ! is_auto_commit_disabled && [[ "$SHOULD_AUTO_COMMIT_CHANGES" == "true" && "${BUILDKITE_PULL_REQUEST:-}" ]]; then
+    if ! is_auto_commit_disabled && [[ "$SHOULD_AUTO_COMMIT_CHANGES" == "true" && "${BUILDKITE_PULL_REQUEST:-false}" != "false" ]]; then
       NEW_COMMIT_MESSAGE="[CI] Auto-commit changed files from '$1'"
       PREVIOUS_COMMIT_MESSAGE="$(git log -1 --pretty=%B)"
 

.buildkite/scripts/steps/artifacts/publish.sh

@@ -23,6 +23,8 @@ download "kibana-$FULL_VERSION-docker-image-aarch64.tar.gz"
 download "kibana-cloud-$FULL_VERSION-docker-image.tar.gz"
 download "kibana-cloud-$FULL_VERSION-docker-image-aarch64.tar.gz"
 download "kibana-ubi-$FULL_VERSION-docker-image.tar.gz"
+download "kibana-wolfi-$FULL_VERSION-docker-image.tar.gz"
+download "kibana-wolfi-$FULL_VERSION-docker-image-aarch64.tar.gz"
 
 download "kibana-$FULL_VERSION-arm64.deb"
 download "kibana-$FULL_VERSION-amd64.deb"
@@ -33,6 +35,7 @@ download "kibana-$FULL_VERSION-docker-build-context.tar.gz"
 download "kibana-cloud-$FULL_VERSION-docker-build-context.tar.gz"
 download "kibana-ironbank-$FULL_VERSION-docker-build-context.tar.gz"
 download "kibana-ubi-$FULL_VERSION-docker-build-context.tar.gz"
+download "kibana-wolfi-$FULL_VERSION-docker-build-context.tar.gz"
 
 download "kibana-$FULL_VERSION-linux-aarch64.tar.gz"
 download "kibana-$FULL_VERSION-linux-x86_64.tar.gz"

.buildkite/scripts/steps/capture_oas_snapshot.sh

@@ -5,7 +5,7 @@ set -euo pipefail
 source .buildkite/scripts/common/util.sh
 
 echo --- Capture OAS snapshot
-cmd="node scripts/capture_oas_snapshot --include-path /api/status"
+cmd="node scripts/capture_oas_snapshot --include-path /api/status --include-path /api/alerting/rule/ --include-path /api/alerting/rules"
 if is_pr && ! is_auto_commit_disabled; then
   cmd="$cmd --update"
 fi

.buildkite/scripts/steps/checks.sh

@@ -15,3 +15,4 @@ fi
 .buildkite/scripts/steps/code_generation/security_solution_codegen.sh
 .buildkite/scripts/steps/openapi_bundling/security_solution_openapi_bundling.sh
 .buildkite/scripts/steps/code_generation/osquery_codegen.sh
+.buildkite/scripts/steps/openapi_bundling/final_merge.sh

.buildkite/scripts/steps/checks/renovate.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+# renovate.json tracks this file for updating the pipelib docker image, update the path if you move this file
+
+set -euo pipefail
+
+echo --- Check renovate.json
+docker run -v "$(pwd)"/renovate.json:/home/app/renovate.json docker.elastic.co/ci-agent-images/pipelib:0.8.0@sha256:641d7fc6cfe473900a1fbe49876762916d804b09fdf2945f74e9f803f3073779 renovate-config-validator

.buildkite/scripts/steps/code_generation/security_solution_codegen.sh

@@ -6,22 +6,19 @@ source .buildkite/scripts/common/util.sh
 
 echo --- Security Solution OpenAPI Code Generation
 
-echo -e "\n[Security Solution OpenAPI Code Generation] OpenAPI Common Package"
-
+echo -e "\n[Security Solution OpenAPI Code Generation] OpenAPI Common Package\n"
 (cd packages/kbn-openapi-common && yarn openapi:generate)
-check_for_changed_files "yarn openapi:generate" true
 
 echo -e "\n[Security Solution OpenAPI Code Generation] Lists Common Package\n"
-
 (cd packages/kbn-securitysolution-lists-common && yarn openapi:generate)
-check_for_changed_files "yarn openapi:generate" true
-
-echo -e "\n[Security Solution OpenAPI Code Generation] Exceptions Common Package"
 
+echo -e "\n[Security Solution OpenAPI Code Generation] Exceptions Common Package\n"
 (cd packages/kbn-securitysolution-exceptions-common && yarn openapi:generate)
-check_for_changed_files "yarn openapi:generate" true
 
-echo -e "\n[Security Solution OpenAPI Code Generation] Security Solution Plugin"
+echo -e "\n[Security Solution OpenAPI Code Generation] Endpoint Exceptions Common Package\n"
+(cd packages/kbn-securitysolution-endpoint-exceptions-common && yarn openapi:generate)
 
+echo -e "\n[Security Solution OpenAPI Code Generation] Security Solution Plugin\n"
 (cd x-pack/plugins/security_solution && yarn openapi:generate)
+
 check_for_changed_files "yarn openapi:generate" true

.buildkite/scripts/steps/openapi_bundling/final_merge.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+echo --- Merge Kibana OpenAPI specs
+
+(cd oas_docs && make api-docs && make api-docs-lint)