Merge branch 'main' into opentelemetry-config-remove-quotes

.backportrc.json

@@ -3,6 +3,7 @@
   "repoName": "kibana",
   "targetBranchChoices": [
     "main",
+    "8.15",
     "8.14",
     "8.13",
     "8.12",
@@ -51,7 +52,7 @@
     "backport"
   ],
   "branchLabelMapping": {
-    "^v8.15.0$": "main",
+    "^v8.16.0$": "main",
     "^v(\\d+).(\\d+).\\d+$": "$1.$2"
   },
   "autoMerge": true,

.buildkite/ftr_configs.yml

@@ -13,10 +13,8 @@ disabled:
   - x-pack/test/security_solution_api_integration/config/ess/config.base.basic.ts
   - x-pack/test/security_solution_api_integration/config/serverless/config.base.ts
   - x-pack/test/security_solution_api_integration/config/serverless/config.base.essentials.ts
-  - x-pack/test/security_solution_api_integration/test_suites/security_solution_endpoint/configs/config.base.ts
   - x-pack/test/security_solution_api_integration/test_suites/security_solution_endpoint_api_int/configs/config.base.ts
-  - x-pack/test/security_solution_endpoint/config.base.ts
-  - x-pack/test/security_solution_endpoint_api_int/config.base.ts
+  - x-pack/test/security_solution_endpoint/configs/config.base.ts
 
   # QA suites that are run out-of-band
   - x-pack/test/stack_functional_integration/configs/config.stack_functional_integration_base.js
@@ -91,6 +89,10 @@ disabled:
   - x-pack/test_serverless/api_integration/test_suites/security/config.feature_flags.ts
   - x-pack/test_serverless/functional/test_suites/security/config.feature_flags.ts
 
+  # http/2 security muted tests
+  - x-pack/test/security_functional/saml.http2.config.ts
+  - x-pack/test/security_functional/oidc.http2.config.ts
+
 defaultQueue: 'n2-4-spot'
 enabled:
   - test/accessibility/config.ts
@@ -401,11 +403,13 @@ enabled:
   - x-pack/test/security_functional/login_selector.config.ts
   - x-pack/test/security_functional/oidc.config.ts
   - x-pack/test/security_functional/saml.config.ts
-  - x-pack/test/security_functional/saml.http2.config.ts
-  - x-pack/test/security_functional/oidc.http2.config.ts
   - x-pack/test/security_functional/insecure_cluster_warning.config.ts
   - x-pack/test/security_functional/user_profiles.config.ts
   - x-pack/test/security_functional/expired_session.config.ts
+  - x-pack/test/security_solution_endpoint/configs/endpoint.config.ts
+  - x-pack/test/security_solution_endpoint/configs/serverless.endpoint.config.ts
+  - x-pack/test/security_solution_endpoint/configs/integrations.config.ts
+  - x-pack/test/security_solution_endpoint/configs/serverless.integrations.config.ts
   - x-pack/test/session_view/basic/config.ts
   - x-pack/test/spaces_api_integration/security_and_spaces/config_basic.ts
   - x-pack/test/spaces_api_integration/security_and_spaces/copy_to_space_config_basic.ts
@@ -578,7 +582,3 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/sources/indices/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/security_solution_endpoint_api_int/configs/config.ts
   - x-pack/test/security_solution_api_integration/test_suites/security_solution_endpoint_api_int/configs/serverless.config.ts
-  - x-pack/test/security_solution_api_integration/test_suites/security_solution_endpoint/configs/endpoint.config.ts
-  - x-pack/test/security_solution_api_integration/test_suites/security_solution_endpoint/configs/serverless.endpoint.config.ts
-  - x-pack/test/security_solution_api_integration/test_suites/security_solution_endpoint/configs/integrations.config.ts
-  - x-pack/test/security_solution_api_integration/test_suites/security_solution_endpoint/configs/serverless.integrations.config.ts

.buildkite/pipeline-resource-definitions/README.md

@@ -0,0 +1,28 @@
+# Buildkite pipeline resource definitions
+
+## Overview
+The pipeline resources are "RRE" (real resource entities) that are used to create/maintain buildkite pipelines.
+
+The resources described in these files are parsed and loaded to Backstage (https://backstage.elastic.dev).
+From there, [Terrazzo](https://buildkite.com/elastic/terrazzo/) is generating and updating the buildkite pipelines.
+
+These pipelines are referenced indirectly through the root's [`catalog-info.yaml`](../../catalog-info.yaml) file in order to reduce bloat in the main resources file.
+There's a location file that collects files defined in this folder ([locations.yml](locations.yml)), this file needs to be updated in order to keep track of local files.
+
+Available parameters and further help can be found here: https://docs.elastic.dev/ci/getting-started-with-buildkite-at-elastic
+
+## Creating a new pipeline resource definition
+The easiest way to create a new pipeline is either by copying and editing a similar pipeline, 
+or by copying a blank template (see [_new_pipeline.yml](_templates/_new_pipeline.yml)) and editing that.
+
+You can validate your pipeline's structural integrity, and it's conformity to baseline rules by running the following command:
+```bash
+.buildkite/pipeline-resource-definitions/scripts/validate-pipeline-definition.sh <path_to_your_pipeline_file>
+```
+
+Once you've added the file, you should update the [locations.yml](locations.yml) file to include the new pipeline, or run the following command to update it:
+```bash
+.buildkite/pipeline-resource-definitions/scripts/fix-location-collection.ts
+```
+
+Add your pipeline implementation, commit & push & merge. The pipeline resource will appear in Backstage within minutes, then the pipeline will be added to Buildkite within ~10 minutes.

.buildkite/pipeline-resource-definitions/kibana-artifacts-snapshot.yml

@@ -22,7 +22,6 @@ spec:
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
         SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
       allow_rebuilds: true
-      branch_configuration: main 8.14 7.17
       default_branch: main
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/artifacts.yml
@@ -44,16 +43,3 @@ spec:
           access_level: MANAGE_BUILD_AND_READ
         kibana-tech-leads:
           access_level: MANAGE_BUILD_AND_READ
-      schedules:
-        Daily build (main):
-          cronline: 0 7 * * * America/New_York
-          message: Daily build
-          branch: main
-        Daily build (8.14):
-          cronline: 0 7 * * * America/New_York
-          message: Daily build
-          branch: '8.14'
-        Daily build (7.17):
-          cronline: 0 7 * * * America/New_York
-          message: Daily build
-          branch: '7.17'

.buildkite/pipeline-resource-definitions/kibana-artifacts-staging.yml

@@ -23,7 +23,6 @@ spec:
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
         SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
       allow_rebuilds: true
-      branch_configuration: 7.17 8.14
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/artifacts.yml
       skip_intermediate_builds: false
@@ -44,12 +43,3 @@ spec:
           access_level: MANAGE_BUILD_AND_READ
         kibana-tech-leads:
           access_level: MANAGE_BUILD_AND_READ
-      schedules:
-        Daily build (8.14):
-          cronline: 0 7 * * * America/New_York
-          message: Daily build
-          branch: '8.14'
-        Daily build (7.17):
-          cronline: 0 7 * * * America/New_York
-          message: Daily build
-          branch: '7.17'

.buildkite/pipeline-resource-definitions/kibana-artifacts-trigger.yml

@@ -23,7 +23,6 @@ spec:
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
         SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
       allow_rebuilds: true
-      branch_configuration: '8.14'
       default_branch: main
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/artifacts_trigger.yml
@@ -45,8 +44,3 @@ spec:
           access_level: MANAGE_BUILD_AND_READ
         kibana-tech-leads:
           access_level: MANAGE_BUILD_AND_READ
-      schedules:
-        Daily build (8.14):
-          cronline: 0 */2 * * * America/New_York
-          message: Daily build
-          branch: '8.14'

.buildkite/pipeline-resource-definitions/kibana-es-forward-testing.yml

@@ -0,0 +1,41 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-es-forward-compatibility-testing
+  description: Forward compatibility testing between Kibana 7.17 and ES 8+
+  links:
+    - url: 'https://buildkite.com/elastic/kibana-es-forward-compatibility-testing'
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  system: buildkite
+  owner: 'group:kibana-operations'
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / ES Forward Compatibility Testing
+      description: Forward compatibility testing between Kibana 7.17 and ES 8+
+    spec:
+      env:
+        SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+      allow_rebuilds: false
+      branch_configuration: main
+      default_branch: main
+      repository: elastic/kibana
+      pipeline_file: .buildkite/pipelines/es_forward.yml # Note: this file exists in 7.17 only
+      skip_intermediate_builds: false
+      provider_settings:
+        prefix_pull_request_fork_branch_names: false
+        trigger_mode: none
+      teams:
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: BUILD_AND_READ

.buildkite/pipeline-resource-definitions/kibana-es-snapshots.yml

@@ -22,7 +22,7 @@ spec:
         SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
       allow_rebuilds: true
-      branch_configuration: main 8.14 7.17
+      branch_configuration: main 8.15 8.14 7.17
       default_branch: main
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/es_snapshots/build.yml
@@ -49,6 +49,10 @@ spec:
           cronline: 0 9 * * * America/New_York
           message: Daily build
           branch: main
+        Daily build (8.15):
+          cronline: 0 9 * * * America/New_York
+          message: Daily build
+          branch: '8.15'
         Daily build (8.14):
           cronline: 0 9 * * * America/New_York
           message: Daily build
@@ -82,7 +86,7 @@ spec:
         SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
       allow_rebuilds: true
-      branch_configuration: main 8.14 7.17
+      branch_configuration: main 8.15 8.14 7.17
       default_branch: main
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/es_snapshots/promote.yml
@@ -130,7 +134,7 @@ spec:
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
         REPORT_FAILED_TESTS_TO_GITHUB: 'true'
       allow_rebuilds: true
-      branch_configuration: main 8.14 7.17
+      branch_configuration: main 8.15 8.14 7.17
       default_branch: main
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/es_snapshots/verify.yml

.buildkite/pipeline-resource-definitions/kibana-on-merge-unsupported-ftrs.yml

@@ -22,7 +22,7 @@ spec:
         SLACK_NOTIFICATIONS_CHANNEL: '#kibana-unsupported-ftrs-alerts'
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
       allow_rebuilds: true
-      branch_configuration: main 8.14 7.17
+      branch_configuration: main 8.15 8.14 7.17
       default_branch: main
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/on_merge_unsupported_ftrs.yml

.buildkite/pipeline-resource-definitions/kibana-on-merge.yml

@@ -25,7 +25,7 @@ spec:
         REPORT_FAILED_TESTS_TO_GITHUB: 'true'
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
       allow_rebuilds: true
-      branch_configuration: main 7.17 8.14
+      branch_configuration: main 7.17 8.14 8.15
       default_branch: main
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/on_merge.yml

.buildkite/pipeline-resource-definitions/kibana-serverless-emergency-release.yml

@@ -0,0 +1,30 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: buildkite-pipeline-kibana-emergency-release
+  description: Emergency release
+  links:
+    - title: Pipeline
+      url: https://buildkite.com/elastic/kibana-emergency-release
+spec:
+  type: buildkite-pipeline
+  owner: group:kibana-operations
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / emergency release
+    spec:
+      repository: elastic/kibana
+      provider_settings:
+        trigger_mode: none
+      pipeline_file: ".buildkite/pipelines/emergency_release.yml"
+      teams:
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-release-operators:
+          access_level: BUILD_AND_READ
+        everyone:
+          access_level: READ_ONLY

.buildkite/pipeline-resource-definitions/kibana-serverless-quality-gates.yml

@@ -0,0 +1,33 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: kibana-tests-pipeline
+  description: Definition of the kibana pipeline
+  links:
+    - title: Pipeline
+      url: https://buildkite.com/elastic/kibana-tests
+spec:
+  type: buildkite-pipeline
+  owner: group:kibana-tech-leads
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana-tests
+      description: Pipeline that tests the service integration in various environments
+    spec:
+      repository: elastic/kibana
+      pipeline_file: ./.buildkite/pipelines/quality-gates/pipeline.kibana-tests.yaml
+      provider_settings:
+        trigger_mode: none
+      teams:
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-release-operators:
+          access_level: BUILD_AND_READ
+        cloud-tooling:
+          access_level: BUILD_AND_READ
+        everyone:
+          access_level: READ_ONLY

.buildkite/pipeline-resource-definitions/locations.yml

@@ -14,6 +14,7 @@ spec:
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-artifacts-staging.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-artifacts-trigger.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-coverage-daily.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-forward-testing.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-serverless-snapshots.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-snapshots.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-esql-grammar-sync.yml
@@ -27,6 +28,8 @@ spec:
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-performance-data-set-extraction-daily.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-pr.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-purge-cloud-deployments.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-emergency-release.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-quality-gates.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-quality-gates-emergency.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-release-testing.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-release.yml
@@ -39,3 +42,4 @@ spec:
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-gen-ai.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-investigations.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-rule-management.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/trigger-version-dependent-jobs.yml

.buildkite/pipeline-resource-definitions/fix-location-collection.ts → .buildkite/pipeline-resource-definitions/scripts/fix-location-collection.ts

@@ -11,7 +11,7 @@ import jsYaml from 'js-yaml';
 import path from 'path';
 import { execSync } from 'child_process';
 
-const EXCLUDE_LIST = ['locations.yml', '_template/template.yml'];
+const EXCLUDE_LIST = ['locations.yml', '_templates'];
 const REPO_FILES_BASE = 'https://github.com/elastic/kibana/blob/main';
 
 type BackstageLocationResource = object & {
@@ -20,19 +20,19 @@ type BackstageLocationResource = object & {
 
 async function main() {
   const repoRoot = execSync('git rev-parse --show-toplevel').toString().trim();
-  const resourceDefinitionsFolder = path.resolve(
+  const resourceDefinitionsRoot = path.resolve(
     repoRoot,
     '.buildkite',
     'pipeline-resource-definitions'
   );
   const resourceDefinitionsBaseUrl = `${REPO_FILES_BASE}/.buildkite/pipeline-resource-definitions`;
-  const locationFile = path.resolve(resourceDefinitionsFolder, 'locations.yml');
+  const locationFile = path.resolve(resourceDefinitionsRoot, 'locations.yml');
   const locationFileLines = fs.readFileSync(locationFile, 'utf8').split('\n');
 
-  const pipelines = readDirRecursively(resourceDefinitionsFolder)
+  const pipelines = readDirRecursively(resourceDefinitionsRoot)
     .filter((file) => file.endsWith('.yml'))
-    .map((file) => file.replace(`${resourceDefinitionsFolder}/`, ''))
-    .filter((f) => !EXCLUDE_LIST.includes(f));
+    .map((file) => file.replace(`${resourceDefinitionsRoot}/`, ''))
+    .filter((f) => EXCLUDE_LIST.every((excludeExpr) => !f.match(excludeExpr)));
 
   const preamble = locationFileLines.slice(0, 1);