Merge remote-tracking branch 'upstream/main' into kbn-24870-fix-redir…

…ect-from-port-host

.buildkite/ftr_configs.yml

@@ -13,10 +13,8 @@ disabled:
   - x-pack/test/security_solution_api_integration/config/ess/config.base.basic.ts
   - x-pack/test/security_solution_api_integration/config/serverless/config.base.ts
   - x-pack/test/security_solution_api_integration/config/serverless/config.base.essentials.ts
-  - x-pack/test/security_solution_api_integration/test_suites/security_solution_endpoint/configs/config.base.ts
   - x-pack/test/security_solution_api_integration/test_suites/security_solution_endpoint_api_int/configs/config.base.ts
-  - x-pack/test/security_solution_endpoint/config.base.ts
-  - x-pack/test/security_solution_endpoint_api_int/config.base.ts
+  - x-pack/test/security_solution_endpoint/configs/config.base.ts
 
   # QA suites that are run out-of-band
   - x-pack/test/stack_functional_integration/configs/config.stack_functional_integration_base.js
@@ -408,6 +406,10 @@ enabled:
   - x-pack/test/security_functional/insecure_cluster_warning.config.ts
   - x-pack/test/security_functional/user_profiles.config.ts
   - x-pack/test/security_functional/expired_session.config.ts
+  - x-pack/test/security_solution_endpoint/configs/endpoint.config.ts
+  - x-pack/test/security_solution_endpoint/configs/serverless.endpoint.config.ts
+  - x-pack/test/security_solution_endpoint/configs/integrations.config.ts
+  - x-pack/test/security_solution_endpoint/configs/serverless.integrations.config.ts
   - x-pack/test/session_view/basic/config.ts
   - x-pack/test/spaces_api_integration/security_and_spaces/config_basic.ts
   - x-pack/test/spaces_api_integration/security_and_spaces/copy_to_space_config_basic.ts
@@ -580,7 +582,3 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/sources/indices/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/security_solution_endpoint_api_int/configs/config.ts
   - x-pack/test/security_solution_api_integration/test_suites/security_solution_endpoint_api_int/configs/serverless.config.ts
-  - x-pack/test/security_solution_api_integration/test_suites/security_solution_endpoint/configs/endpoint.config.ts
-  - x-pack/test/security_solution_api_integration/test_suites/security_solution_endpoint/configs/serverless.endpoint.config.ts
-  - x-pack/test/security_solution_api_integration/test_suites/security_solution_endpoint/configs/integrations.config.ts
-  - x-pack/test/security_solution_api_integration/test_suites/security_solution_endpoint/configs/serverless.integrations.config.ts

.buildkite/pipeline-resource-definitions/README.md

@@ -0,0 +1,28 @@
+# Buildkite pipeline resource definitions
+
+## Overview
+The pipeline resources are "RRE" (real resource entities) that are used to create/maintain buildkite pipelines.
+
+The resources described in these files are parsed and loaded to Backstage (https://backstage.elastic.dev).
+From there, [Terrazzo](https://buildkite.com/elastic/terrazzo/) is generating and updating the buildkite pipelines.
+
+These pipelines are referenced indirectly through the root's [`catalog-info.yaml`](../../catalog-info.yaml) file in order to reduce bloat in the main resources file.
+There's a location file that collects files defined in this folder ([locations.yml](locations.yml)), this file needs to be updated in order to keep track of local files.
+
+Available parameters and further help can be found here: https://docs.elastic.dev/ci/getting-started-with-buildkite-at-elastic
+
+## Creating a new pipeline resource definition
+The easiest way to create a new pipeline is either by copying and editing a similar pipeline, 
+or by copying a blank template (see [_new_pipeline.yml](_templates/_new_pipeline.yml)) and editing that.
+
+You can validate your pipeline's structural integrity, and it's conformity to baseline rules by running the following command:
+```bash
+.buildkite/pipeline-resource-definitions/scripts/validate-pipeline-definition.sh <path_to_your_pipeline_file>
+```
+
+Once you've added the file, you should update the [locations.yml](locations.yml) file to include the new pipeline, or run the following command to update it:
+```bash
+.buildkite/pipeline-resource-definitions/scripts/fix-location-collection.ts
+```
+
+Add your pipeline implementation, commit & push & merge. The pipeline resource will appear in Backstage within minutes, then the pipeline will be added to Buildkite within ~10 minutes.

.buildkite/pipeline-resource-definitions/kibana-artifacts-snapshot.yml

@@ -22,7 +22,6 @@ spec:
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
         SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
       allow_rebuilds: true
-      branch_configuration: main 8.15 8.14 7.17
       default_branch: main
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/artifacts.yml
@@ -44,20 +43,3 @@ spec:
           access_level: MANAGE_BUILD_AND_READ
         kibana-tech-leads:
           access_level: MANAGE_BUILD_AND_READ
-      schedules:
-        Daily build (main):
-          cronline: 0 7 * * * America/New_York
-          message: Daily build
-          branch: main
-        Daily build (8.15):
-          cronline: 0 7 * * * America/New_York
-          message: Daily build
-          branch: '8.15'
-        Daily build (8.14):
-          cronline: 0 7 * * * America/New_York
-          message: Daily build
-          branch: '8.14'
-        Daily build (7.17):
-          cronline: 0 7 * * * America/New_York
-          message: Daily build
-          branch: '7.17'

.buildkite/pipeline-resource-definitions/kibana-artifacts-staging.yml

@@ -23,7 +23,6 @@ spec:
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
         SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
       allow_rebuilds: true
-      branch_configuration: 7.17 8.14 8.15
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/artifacts.yml
       skip_intermediate_builds: false
@@ -44,16 +43,3 @@ spec:
           access_level: MANAGE_BUILD_AND_READ
         kibana-tech-leads:
           access_level: MANAGE_BUILD_AND_READ
-      schedules:
-        Daily build (8.15):
-          cronline: 0 7 * * * America/New_York
-          message: Daily build
-          branch: '8.15'
-        Daily build (8.14):
-          cronline: 0 7 * * * America/New_York
-          message: Daily build
-          branch: '8.14'
-        Daily build (7.17):
-          cronline: 0 7 * * * America/New_York
-          message: Daily build
-          branch: '7.17'

.buildkite/pipeline-resource-definitions/kibana-artifacts-trigger.yml

@@ -23,7 +23,6 @@ spec:
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
         SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
       allow_rebuilds: true
-      branch_configuration: 8.15 8.14
       default_branch: main
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/artifacts_trigger.yml
@@ -45,12 +44,3 @@ spec:
           access_level: MANAGE_BUILD_AND_READ
         kibana-tech-leads:
           access_level: MANAGE_BUILD_AND_READ
-      schedules:
-        Daily build (8.15):
-          cronline: 0 */2 * * * America/New_York
-          message: Daily build
-          branch: '8.15'
-        Daily build (8.14):
-          cronline: 0 */2 * * * America/New_York
-          message: Daily build
-          branch: '8.14'

.buildkite/pipeline-resource-definitions/kibana-es-forward-testing.yml

@@ -0,0 +1,41 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-es-forward-compatibility-testing
+  description: Forward compatibility testing between Kibana 7.17 and ES 8+
+  links:
+    - url: 'https://buildkite.com/elastic/kibana-es-forward-compatibility-testing'
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  system: buildkite
+  owner: 'group:kibana-operations'
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / ES Forward Compatibility Testing
+      description: Forward compatibility testing between Kibana 7.17 and ES 8+
+    spec:
+      env:
+        SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+      allow_rebuilds: false
+      branch_configuration: main
+      default_branch: main
+      repository: elastic/kibana
+      pipeline_file: .buildkite/pipelines/es_forward.yml # Note: this file exists in 7.17 only
+      skip_intermediate_builds: false
+      provider_settings:
+        prefix_pull_request_fork_branch_names: false
+        trigger_mode: none
+      teams:
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: BUILD_AND_READ

.buildkite/pipeline-resource-definitions/locations.yml

@@ -14,6 +14,7 @@ spec:
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-artifacts-staging.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-artifacts-trigger.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-coverage-daily.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-forward-testing.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-serverless-snapshots.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-snapshots.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-esql-grammar-sync.yml
@@ -41,3 +42,4 @@ spec:
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-gen-ai.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-investigations.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-rule-management.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/trigger-version-dependent-jobs.yml

.buildkite/pipeline-resource-definitions/fix-location-collection.ts → .buildkite/pipeline-resource-definitions/scripts/fix-location-collection.ts

@@ -11,7 +11,7 @@ import jsYaml from 'js-yaml';
 import path from 'path';
 import { execSync } from 'child_process';
 
-const EXCLUDE_LIST = ['locations.yml', '_template/template.yml'];
+const EXCLUDE_LIST = ['locations.yml', '_templates'];
 const REPO_FILES_BASE = 'https://github.com/elastic/kibana/blob/main';
 
 type BackstageLocationResource = object & {
@@ -20,19 +20,19 @@ type BackstageLocationResource = object & {
 
 async function main() {
   const repoRoot = execSync('git rev-parse --show-toplevel').toString().trim();
-  const resourceDefinitionsFolder = path.resolve(
+  const resourceDefinitionsRoot = path.resolve(
     repoRoot,
     '.buildkite',
     'pipeline-resource-definitions'
   );
   const resourceDefinitionsBaseUrl = `${REPO_FILES_BASE}/.buildkite/pipeline-resource-definitions`;
-  const locationFile = path.resolve(resourceDefinitionsFolder, 'locations.yml');
+  const locationFile = path.resolve(resourceDefinitionsRoot, 'locations.yml');
   const locationFileLines = fs.readFileSync(locationFile, 'utf8').split('\n');
 
-  const pipelines = readDirRecursively(resourceDefinitionsFolder)
+  const pipelines = readDirRecursively(resourceDefinitionsRoot)
     .filter((file) => file.endsWith('.yml'))
-    .map((file) => file.replace(`${resourceDefinitionsFolder}/`, ''))
-    .filter((f) => !EXCLUDE_LIST.includes(f));
+    .map((file) => file.replace(`${resourceDefinitionsRoot}/`, ''))
+    .filter((f) => EXCLUDE_LIST.every((excludeExpr) => !f.match(excludeExpr)));
 
   const preamble = locationFileLines.slice(0, 1);
 

.buildkite/pipeline-resource-definitions/scripts/validate-pipeline-definition.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+# This script is used to validate a single RRE for a pipeline definition.
+
+TARGET_FILE=$1
+
+if [ -z "$TARGET_FILE" ]; then
+  echo "Usage: $0 <path_to_your_pipeline_file>"
+  exit 1
+fi
+
+echo "Validating $TARGET_FILE..."
+ABSOLUTE_PATH=$(realpath "$TARGET_FILE")
+FILE_NAME=$(basename "$ABSOLUTE_PATH")
+FOLDER_NAME=$(dirname "$ABSOLUTE_PATH")
+
+docker run -it \
+ --mount type=bind,source="$FOLDER_NAME",target=/home/app/ \
+  docker.elastic.co/ci-agent-images/pipelib \
+  rre validate --backstage-entity-aware "/home/app/$FILE_NAME"
+
+if [ $? -ne 0 ]; then
+  echo "$FILE_NAME invalid ❌"
+  exit 1
+else
+  echo "$FILE_NAME valid ✅"
+  exit 0
+fi

.buildkite/pipeline-resource-definitions/trigger-version-dependent-jobs.yml

@@ -0,0 +1,71 @@
+###
+# For more information on authoring pipeline definitions,
+# follow the guides at https://docs.elastic.dev/ci/getting-started-with-buildkite-at-elastic
+###
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-trigger-version-dependent-jobs
+  description: 'Trigger version-dependent jobs'
+  links:
+    - url: 'https://buildkite.com/elastic/kibana-trigger-version-dependent-jobs'
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  system: buildkite
+  owner: 'group:kibana-operations'
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / trigger version-dependent jobs
+      description: 'Trigger version-dependent jobs'
+    spec:
+      env:
+        SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+
+      allow_rebuilds: false
+      branch_configuration: main
+      default_branch: main
+      repository: elastic/kibana
+      pipeline_file: .buildkite/scripts/pipelines/trigger_version_dependent_jobs/pipeline.sh
+      skip_intermediate_builds: false
+      provider_settings:
+        prefix_pull_request_fork_branch_names: false
+        skip_pull_request_builds_for_existing_commits: true
+        trigger_mode: none
+      teams:
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: BUILD_AND_READ
+      schedules:
+        Trigger ES forward compatibility tests:
+          cronline: 0 5 * * *
+          message: Trigger ES forward compatibility tests
+          env:
+            TRIGGER_PIPELINE_SET: es-forward
+        Trigger artifact staging builds:
+          cronline: 0 7 * * * America/New_York
+          message: Trigger artifact staging builds
+          env:
+            TRIGGER_PIPELINE_SET: artifacts-staging
+            MESSAGE: Daily staging build
+        Trigger artifact snapshot builds:
+          cronline: 0 7 * * * America/New_York
+          message: Trigger artifact snapshot builds
+          env:
+              TRIGGER_PIPELINE_SET: artifacts-snapshot
+              MESSAGE: Daily snapshot build
+        Run kibana-artifacts-trigger:
+          cronline: 0 */2 * * * America/New_York
+          message: Trigger 'kibana-artifacts-trigger'
+          env:
+            TRIGGER_PIPELINE_SET: artifacts-trigger
+            MESSAGE: Daily build

.buildkite/pipeline-utils/agent_images.ts

@@ -52,4 +52,19 @@ function getAgentImageConfig({ returnYaml = false } = {}): string | AgentImageCo
   return config;
 }
 
-export { getAgentImageConfig };
+const expandAgentQueue = (queueName: string = 'n2-4-spot') => {
+  const [kind, cores, addition] = queueName.split('-');
+  const additionalProps =
+    {
+      spot: { preemptible: true },
+      virt: { localSsdInterface: 'nvme', enableNestedVirtualization: true, localSsds: 1 },
+    }[addition] || {};
+
+  return {
+    ...getAgentImageConfig(),
+    machineType: `${kind}-standard-${cores}`,
+    ...additionalProps,
+  };
+};
+
+export { getAgentImageConfig, expandAgentQueue };