Merge remote-tracking branch 'upstream/main' into remove-log-stream-a…

…nd-settings
elastic · Dec 31, 2024 · a37a782 · a37a782
2 parents be1e2b7 + 38310a5
commit a37a782
Show file tree

Hide file tree

Showing 48,106 changed files with 350,004 additions and 325,979 deletions.
diff --git a/.buildkite/ftr_oblt_stateful_configs.yml b/.buildkite/ftr_oblt_stateful_configs.yml
@@ -1,15 +1,15 @@
 disabled:
   # Cypress configs, for now these are still run manually
-  - x-pack/plugins/observability_solution/observability_onboarding/e2e/ftr_config_open.ts
-  - x-pack/plugins/observability_solution/observability_onboarding/e2e/ftr_config_runner.ts
-  - x-pack/plugins/observability_solution/observability_onboarding/e2e/ftr_config.ts
-  - x-pack/plugins/observability_solution/apm/ftr_e2e/ftr_config_run.ts
-  - x-pack/plugins/observability_solution/apm/ftr_e2e/ftr_config.ts
-  - x-pack/plugins/observability_solution/inventory/e2e/ftr_config_run.ts
-  - x-pack/plugins/observability_solution/inventory/e2e/ftr_config.ts
-  - x-pack/plugins/observability_solution/profiling/e2e/ftr_config_open.ts
-  - x-pack/plugins/observability_solution/profiling/e2e/ftr_config_runner.ts
-  - x-pack/plugins/observability_solution/profiling/e2e/ftr_config.ts
+  - x-pack/solutions/observability/plugins/observability_onboarding/e2e/ftr_config_open.ts
+  - x-pack/solutions/observability/plugins/observability_onboarding/e2e/ftr_config_runner.ts
+  - x-pack/solutions/observability/plugins/observability_onboarding/e2e/ftr_config.ts
+  - x-pack/solutions/observability/plugins/apm/ftr_e2e/ftr_config_run.ts
+  - x-pack/solutions/observability/plugins/apm/ftr_e2e/ftr_config.ts
+  - x-pack/solutions/observability/plugins/inventory/e2e/ftr_config_run.ts
+  - x-pack/solutions/observability/plugins/inventory/e2e/ftr_config.ts
+  - x-pack/solutions/observability/plugins/profiling/e2e/ftr_config_open.ts
+  - x-pack/solutions/observability/plugins/profiling/e2e/ftr_config_runner.ts
+  - x-pack/solutions/observability/plugins/profiling/e2e/ftr_config.ts
 
   #FTR configs
   - x-pack/solutions/observability/plugins/uptime/e2e/config.ts
@@ -20,7 +20,7 @@ disabled:
   - x-pack/solutions/observability/plugins/synthetics/e2e/synthetics/synthetics_run.ts
   - x-pack/solutions/observability/plugins/exploratory_view/e2e/synthetics_run.ts
   - x-pack/solutions/observability/plugins/ux/e2e/synthetics_run.ts
-  - x-pack/plugins/observability_solution/slo/e2e/synthetics_run.ts
+  - x-pack/solutions/observability/plugins/slo/e2e/synthetics_run.ts
 
 defaultQueue: 'n2-4-spot'
 enabled:

diff --git a/.buildkite/ftr_platform_stateful_configs.yml b/.buildkite/ftr_platform_stateful_configs.yml
@@ -131,6 +131,7 @@ enabled:
   - x-pack/test/alerting_api_integration/security_and_spaces/group2/config.ts
   - x-pack/test/alerting_api_integration/security_and_spaces/group3/config.ts
   - x-pack/test/alerting_api_integration/security_and_spaces/group4/config.ts
+  - x-pack/test/alerting_api_integration/security_and_spaces/group5/config.ts
   - x-pack/test/alerting_api_integration/security_and_spaces/group3/config_with_schedule_circuit_breaker.ts
   - x-pack/test/alerting_api_integration/security_and_spaces/group2/config_non_dedicated_task_runner.ts
   - x-pack/test/alerting_api_integration/security_and_spaces/group4/config_non_dedicated_task_runner.ts

diff --git a/.buildkite/ftr_security_serverless_configs.yml b/.buildkite/ftr_security_serverless_configs.yml
@@ -89,6 +89,7 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_read/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_read/basic_license_essentials_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/telemetry/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/user_roles/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/genai/nlp_cleanup_task/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/genai/nlp_cleanup_task/basic_license_essentials_tier/configs/serverless.config.ts

diff --git a/.buildkite/ftr_security_stateful_configs.yml b/.buildkite/ftr_security_stateful_configs.yml
@@ -70,6 +70,7 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_read/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_read/basic_license_essentials_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/telemetry/trial_license_complete_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/telemetry/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/user_roles/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/risk_engine/basic_license_essentials_tier/configs/ess.config.ts

diff --git a/.buildkite/pipeline-resource-definitions/kibana-es-forward-testing-9-dot-0.yml b/.buildkite/pipeline-resource-definitions/kibana-es-forward-testing-9-dot-0.yml
@@ -27,6 +27,7 @@ spec:
       default_branch: main
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/es_forward_9_dot_0.yml # Note: this file exists in 8.x only and should be moved into 8.18 once the branch is cut
+      skip_intermediate_builds: false
       provider_settings:
         prefix_pull_request_fork_branch_names: false
         trigger_mode: none

diff --git a/.buildkite/pipeline-resource-definitions/kibana-es-forward-testing.yml b/.buildkite/pipeline-resource-definitions/kibana-es-forward-testing.yml
@@ -27,6 +27,7 @@ spec:
       default_branch: main
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/es_forward.yml # Note: this file exists in 7.17 only
+      skip_intermediate_builds: false
       provider_settings:
         prefix_pull_request_fork_branch_names: false
         trigger_mode: none

diff --git a/.buildkite/pipeline-resource-definitions/kibana-package-registry.yml b/.buildkite/pipeline-resource-definitions/kibana-package-registry.yml
@@ -2,10 +2,10 @@
 apiVersion: backstage.io/v1alpha1
 kind: Resource
 metadata:
-  name: bk-kibana-package-registry-promote
+  name: bk-kibana-package-registry-verify-and-promote
   description: Promote package-registry/distribution:lite
   links:
-    - url: 'https://buildkite.com/elastic/kibana-package-registry-promote'
+    - url: 'https://buildkite.com/elastic/kibana-package-registry-verify-and-promote'
       title: Pipeline link
 spec:
   type: buildkite-pipeline
@@ -15,18 +15,21 @@ spec:
     apiVersion: buildkite.elastic.dev/v1
     kind: Pipeline
     metadata:
-      name: kibana / package registry promote
-      description: Promote package-registry/distribution:lite
+      name: kibana / package registry verify and promote
+      description: Verify and promote docker.elastic.co/package-registry/distribution:lite
     spec:
       env:
         SLACK_NOTIFICATIONS_CHANNEL: "#kibana-operations-alerts"
-        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "false"
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "true"
       repository: elastic/kibana
       branch_configuration: main
       default_branch: main
       pipeline_file: ".buildkite/pipelines/fleet/package_registry.yml"
       provider_settings:
         trigger_mode: none
+      schedules:
+        Daily promotion:
+          cronline: 0 22 * * * America/New_York
       teams:
         everyone:
           access_level: BUILD_AND_READ

diff --git a/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts b/.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts
@@ -485,7 +485,10 @@ export async function pickTestGroupRunOrder() {
             parallelism: unit.count,
             timeout_in_minutes: 120,
             key: 'jest',
-            agents: expandAgentQueue('n2-4-spot'),
+            agents: {
+              ...expandAgentQueue('n2-4-spot'),
+              diskSizeGb: 75,
+            },
             retry: {
               automatic: [
                 { exit_status: '-1', limit: 3 },

diff --git a/.buildkite/pipelines/fleet/package_registry.yml b/.buildkite/pipelines/fleet/package_registry.yml
@@ -1,2 +1,106 @@
+env:
+  IGNORE_SHIP_CI_STATS_ERROR: 'true'
 steps:
-  - command: echo "Placeholder"
+  - command: .buildkite/scripts/lifecycle/pre_build.sh
+    label: Pre-Build
+    timeout_in_minutes: 10
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2
+
+  - wait
+
+  - command: .buildkite/scripts/steps/build_kibana.sh
+    label: Build Kibana Distribution
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-8
+      preemptible: true
+    key: build
+    if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
+    timeout_in_minutes: 60
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
+
+  - command: .buildkite/scripts/steps/test/pick_test_group_run_order.sh
+    label: 'Pick Test Group Run Order'
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2
+    timeout_in_minutes: 10
+    env:
+      JEST_UNIT_SCRIPT: '.buildkite/scripts/steps/test/jest.sh'
+      JEST_INTEGRATION_SCRIPT: '.buildkite/scripts/steps/test/jest_integration.sh'
+      FTR_CONFIGS_SCRIPT: '.buildkite/scripts/steps/test/ftr_configs.sh'
+      LIMIT_CONFIG_TYPE: integration,functional
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/defend_workflows.sh
+    label: 'Defend Workflows Cypress Tests'
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      enableNestedVirtualization: true
+      machineType: n2-standard-4
+    timeout_in_minutes: 60
+    parallelism: 20
+    key: defend-workflows-stateful
+    depends_on: build
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/defend_workflows_serverless.sh
+    label: 'Defend Workflows Cypress Tests on Serverless'
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      enableNestedVirtualization: true
+      machineType: n2-standard-4
+    timeout_in_minutes: 60
+    parallelism: 14
+    key: defend-workflows-serverless
+    depends_on: build
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/fleet/promote_package_registry.sh
+    label: Promote docker.elastic.co/package-registry/distribution:lite
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2
+    depends_on:
+      - ftr-configs
+      - jest-integration
+      - defend-worfklows-stateful
+      - defend-workflows-serverless
+
+  - wait: ~
+    continue_on_failure: true
+
+  - command: .buildkite/scripts/lifecycle/post_build.sh
+    label: Post-Build
+    timeout_in_minutes: 10
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-2
diff --git a/.buildkite/pipelines/on_merge.yml b/.buildkite/pipelines/on_merge.yml
@@ -10,6 +10,7 @@ steps:
       imageProject: elastic-images-prod
       provider: gcp
       machineType: n2-standard-2
+      diskSizeGb: 75
     retry:
       automatic:
         - exit_status: '*'
@@ -39,6 +40,7 @@ steps:
       provider: gcp
       machineType: n2-highcpu-8
       preemptible: true
+      diskSizeGb: 75
     timeout_in_minutes: 60
     retry:
       automatic:
@@ -53,6 +55,7 @@ steps:
       provider: gcp
       machineType: n2-standard-2
       preemptible: true
+      diskSizeGb: 75
     timeout_in_minutes: 60
     retry:
       automatic:
@@ -67,6 +70,7 @@ steps:
       provider: gcp
       machineType: n2-standard-16
       preemptible: true
+      diskSizeGb: 75
     timeout_in_minutes: 60
     retry:
       automatic:
@@ -81,6 +85,7 @@ steps:
       provider: gcp
       machineType: n2-standard-32
       preemptible: true
+      diskSizeGb: 75
     timeout_in_minutes: 60
     retry:
       automatic:
@@ -97,6 +102,7 @@ steps:
       diskType: 'hyperdisk-balanced'
       preemptible: true
       spotZones: us-central1-a,us-central1-b,us-central1-c
+      diskSizeGb: 75
     timeout_in_minutes: 60
     retry:
       automatic:
@@ -111,6 +117,7 @@ steps:
       provider: gcp
       machineType: n2-standard-4
       preemptible: true
+      diskSizeGb: 75
     timeout_in_minutes: 60
     retry:
       automatic:
@@ -128,6 +135,7 @@ steps:
       provider: gcp
       machineType: n2-standard-4
       preemptible: true
+      diskSizeGb: 75
     timeout_in_minutes: 80
     retry:
       automatic:
@@ -143,6 +151,7 @@ steps:
       imageProject: elastic-images-prod
       provider: gcp
       machineType: n2-standard-2
+      diskSizeGb: 75
     timeout_in_minutes: 10
     depends_on:
       - build
@@ -159,6 +168,7 @@ steps:
       imageProject: elastic-images-prod
       provider: gcp
       machineType: n2-standard-2
+      diskSizeGb: 75
     timeout_in_minutes: 10
     env:
       JEST_UNIT_SCRIPT: '.buildkite/scripts/steps/test/jest.sh'

diff --git a/.buildkite/pipelines/pull_request/base.yml b/.buildkite/pipelines/pull_request/base.yml
@@ -4,6 +4,7 @@ steps:
     timeout_in_minutes: 10
     agents:
       machineType: n2-standard-2
+      diskSizeGb: 75
 
   - wait
 
@@ -26,6 +27,7 @@ steps:
     agents:
       machineType: n2-highcpu-8
       preemptible: true
+      diskSizeGb: 75
     key: quick_checks
     timeout_in_minutes: 60
     retry:
@@ -39,6 +41,7 @@ steps:
     agents:
       machineType: n2-standard-2
       preemptible: true
+      diskSizeGb: 75
     timeout_in_minutes: 60
     retry:
       automatic:
@@ -50,6 +53,7 @@ steps:
     agents:
       machineType: n2-standard-16
       preemptible: true
+      diskSizeGb: 75
     key: linting
     timeout_in_minutes: 60
     retry:
@@ -62,6 +66,7 @@ steps:
     agents:
       machineType: n2-standard-32
       preemptible: true
+      diskSizeGb: 75
     key: linting_with_types
     timeout_in_minutes: 60
     retry:
@@ -74,6 +79,7 @@ steps:
     agents:
       machineType: n2-standard-4
       preemptible: true
+      diskSizeGb: 75
     key: check_oas_snapshot
     timeout_in_minutes: 60
     retry:
@@ -88,6 +94,7 @@ steps:
       diskType: 'hyperdisk-balanced'
       preemptible: true
       spotZones: us-central1-a,us-central1-b,us-central1-c
+      diskSizeGb: 75
     key: check_types
     timeout_in_minutes: 60
     retry:
@@ -101,6 +108,7 @@ steps:
     label: Mark CI Stats as ready
     agents:
       machineType: n2-standard-2
+      diskSizeGb: 75
     timeout_in_minutes: 10
     depends_on:
       - build
@@ -114,6 +122,7 @@ steps:
     label: 'Pick Test Group Run Order'
     agents:
       machineType: n2-standard-2
+      diskSizeGb: 75
     timeout_in_minutes: 10
     env:
       JEST_UNIT_SCRIPT: '.buildkite/scripts/steps/test/jest.sh'
@@ -129,6 +138,7 @@ steps:
     agents:
       machineType: n2-standard-4
       preemptible: true
+      diskSizeGb: 75
     key: build_api_docs
     timeout_in_minutes: 90
     retry: