Merge branch 'main' of github.com:elastic/kibana into config-schema-a…

…s-server-only

.buildkite/ftr_oblt_serverless_configs.yml

@@ -27,4 +27,4 @@ enabled:
   - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group6.ts
   - x-pack/test_serverless/functional/test_suites/observability/config.screenshots.ts
   # serverless config files that run deployment-agnostic tests
-  - x-pack/test/api_integration/deployment_agnostic/oblt.serverless.config.ts
+  - x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts

.buildkite/ftr_oblt_stateful_configs.yml

@@ -25,6 +25,7 @@ defaultQueue: 'n2-4-spot'
 enabled:
   - x-pack/test/alerting_api_integration/observability/config.ts
   - x-pack/test/api_integration/apis/logs_ui/config.ts
+  - x-pack/test/api_integration/apis/logs_shared/config.ts
   - x-pack/test/api_integration/apis/metrics_ui/config.ts
   - x-pack/test/api_integration/apis/osquery/config.ts
   - x-pack/test/api_integration/apis/synthetics/config.ts
@@ -49,3 +50,5 @@ enabled:
   - x-pack/test/observability_ai_assistant_functional/enterprise/config.ts
   - x-pack/test/profiling_api_integration/cloud/config.ts
   - x-pack/test/functional/apps/apm/config.ts
+  # stateful configs that run deployment-agnostic tests
+  - x-pack/test/api_integration/deployment_agnostic/configs/stateful/oblt.stateful.config.ts

.buildkite/ftr_platform_stateful_configs.yml

@@ -272,6 +272,7 @@ enabled:
   - x-pack/test/functional/config.firefox.js
   - x-pack/test/functional/config.upgrade_assistant.ts
   - x-pack/test/functional_cloud/config.ts
+  - x-pack/test/functional_solution_sidenav/config.ts
   - x-pack/test/kubernetes_security/basic/config.ts
   - x-pack/test/licensing_plugin/config.public.ts
   - x-pack/test/licensing_plugin/config.ts
@@ -361,4 +362,4 @@ enabled:
   - x-pack/performance/journeys_e2e/infra_hosts_view.ts
   - x-pack/test/custom_branding/config.ts
   # stateful config files that run deployment-agnostic tests
-  - x-pack/test/api_integration/deployment_agnostic/stateful.config.ts
+  - x-pack/test/api_integration/deployment_agnostic/configs/stateful/platform.stateful.config.ts

.buildkite/ftr_search_serverless_configs.yml

@@ -17,4 +17,4 @@ enabled:
   - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group5.ts
   - x-pack/test_serverless/functional/test_suites/search/common_configs/config.group6.ts
   # serverless config files that run deployment-agnostic tests
-  - x-pack/test/api_integration/deployment_agnostic/search.serverless.config.ts
+  - x-pack/test/api_integration/deployment_agnostic/configs/serverless/search.serverless.config.ts

.buildkite/ftr_security_serverless_configs.yml

@@ -35,6 +35,7 @@ enabled:
   - x-pack/test_serverless/functional/test_suites/security/config.cloud_security_posture.basic.ts
   - x-pack/test_serverless/functional/test_suites/security/config.cloud_security_posture.essentials.ts
   - x-pack/test_serverless/functional/test_suites/security/config.cloud_security_posture.agentless.ts
+  - x-pack/test_serverless/functional/test_suites/security/config.cloud_security_posture.agentless_api.ts
   - x-pack/test_serverless/functional/test_suites/security/config.saved_objects_management.ts
   - x-pack/test_serverless/functional/test_suites/security/config.context_awareness.ts
   - x-pack/test_serverless/functional/test_suites/security/common_configs/config.group1.ts
@@ -98,4 +99,4 @@ enabled:
   - x-pack/test/security_solution_endpoint/configs/serverless.endpoint.config.ts
   - x-pack/test/security_solution_endpoint/configs/serverless.integrations.config.ts
   # serverless config files that run deployment-agnostic tests
-  - x-pack/test/api_integration/deployment_agnostic/security.serverless.config.ts
+  - x-pack/test/api_integration/deployment_agnostic/configs/serverless/security.serverless.config.ts

.buildkite/package.json

@@ -11,7 +11,7 @@
   },
   "dependencies": {
     "@octokit/rest": "^18.10.0",
-    "axios": "^1.6.3",
+    "axios": "^1.7.4",
     "globby": "^11.1.0",
     "js-yaml": "^4.1.0",
     "minimatch": "^5.0.1",

.buildkite/pipeline-resource-definitions/kibana-codeql.yml

@@ -0,0 +1,34 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-codeql
+  description: Run CodeQL
+  links:
+    - title: Pipeline link
+      url: https://buildkite.com/elastic/kibana-codeql
+spec:
+  type: buildkite-pipeline
+  owner: group:kibana-operations
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / codeql
+      description: Run CodeQL
+    spec:
+      env:
+        SLACK_NOTIFICATIONS_CHANNEL: "#kibana-operations-alerts"
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "false"
+      repository: elastic/kibana
+      branch_configuration: main
+      default_branch: main
+      pipeline_file: ".buildkite/pipelines/codeql/codeql.yml"
+      provider_settings:
+        trigger_mode: none
+      teams:
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: READ_ONLY

.buildkite/pipeline-resource-definitions/locations.yml

@@ -44,3 +44,4 @@ spec:
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-rule-management.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/trigger-version-dependent-jobs.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-pointer-compression.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-codeql.yml

.buildkite/pipeline-resource-definitions/scripts/fix-location-collection.ts

@@ -36,13 +36,15 @@ async function main() {
 
   const preamble = locationFileLines.slice(0, 1);
 
+  // eslint-disable-next-line @kbn/eslint/no_unsafe_js_yaml
   const locationObj = jsYaml.load(
     locationFileLines.slice(1).join('\n')
   ) as BackstageLocationResource;
   locationObj.spec.targets = pipelines.map(
     (fileName) => `${resourceDefinitionsBaseUrl}/${fileName}`
   );
 
+  // eslint-disable-next-line @kbn/eslint/no_unsafe_js_yaml
   const locationYaml = jsYaml.dump(locationObj, { lineWidth: 400 });
 
   fs.writeFileSync(locationFile, `${preamble.join('\n')}\n${locationYaml}`);

.buildkite/pipeline-resource-definitions/scripts/validate-pipeline-definition.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 
+# renovate.json tracks this file for updating the pipelib docker image, update the path if you move this file
 # This script is used to validate a single RRE for a pipeline definition.
 
 TARGET_FILE=$1
@@ -16,7 +17,7 @@ FOLDER_NAME=$(dirname "$ABSOLUTE_PATH")
 
 docker run -it \
  --mount type=bind,source="$FOLDER_NAME",target=/home/app/ \
-  docker.elastic.co/ci-agent-images/pipelib \
+  docker.elastic.co/ci-agent-images/pipelib:0.8.0@sha256:641d7fc6cfe473900a1fbe49876762916d804b09fdf2945f74e9f803f3073779 \
   rre validate --backstage-entity-aware "/home/app/$FILE_NAME"
 
 if [ $? -ne 0 ]; then

.buildkite/pipeline-utils/agent_images.ts

@@ -6,6 +6,7 @@
  * Side Public License, v 1.
  */
 
+// eslint-disable-next-line @kbn/eslint/no_unsafe_js_yaml
 import { dump } from 'js-yaml';
 import { BuildkiteClient, BuildkiteCommandStep } from './buildkite';
 

.buildkite/pipeline-utils/buildkite/client.ts

@@ -8,7 +8,10 @@
 
 import axios, { AxiosInstance } from 'axios';
 import { execSync, ExecSyncOptions } from 'child_process';
+
+// eslint-disable-next-line @kbn/eslint/no_unsafe_js_yaml
 import { dump } from 'js-yaml';
+
 import { parseLinkHeader } from './parse_link_header';
 import { Artifact } from './types/artifact';
 import { Build, BuildStatus } from './types/build';

.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts

@@ -10,6 +10,8 @@ import * as Fs from 'fs';
 
 import * as globby from 'globby';
 import minimatch from 'minimatch';
+
+// eslint-disable-next-line @kbn/eslint/no_unsafe_js_yaml
 import { load as loadYaml } from 'js-yaml';
 
 import { BuildkiteClient, BuildkiteStep } from '../buildkite';

.buildkite/pipelines/codeql/codeql.yml

@@ -0,0 +1,2 @@
+steps:
+  - command: echo "Placeholder"

.buildkite/pipelines/on_merge.yml

@@ -37,7 +37,7 @@ steps:
       image: family/kibana-ubuntu-2004
       imageProject: elastic-images-prod
       provider: gcp
-      machineType: n2-standard-2
+      machineType: n2-highcpu-8
       preemptible: true
     key: quick_checks
     timeout_in_minutes: 60
@@ -589,6 +589,11 @@ steps:
       preemptible: true
     artifact_paths:
       "target/plugin_so_types_snapshot.json"
+    timeout_in_minutes: 30
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
 
   - wait: ~
     continue_on_failure: true

.buildkite/pipelines/pull_request/base.yml

@@ -23,7 +23,7 @@ steps:
   - command: .buildkite/scripts/steps/quick_checks.sh
     label: 'Quick Checks'
     agents:
-      machineType: n2-standard-2
+      machineType: n2-highcpu-8
       preemptible: true
     key: quick_checks
     timeout_in_minutes: 60

.buildkite/pipelines/security_solution_quality_gate/mki_periodic/mki_periodic_defend_workflows.yml

@@ -2,7 +2,7 @@ steps:
   - group: "Cypress MKI - Defend Workflows"
     key: cypress_test_defend_workflows
     steps:
-      - label: "Running cypress:dw:qa:serverless:run"
+      - label: "Cypress - DW - Running cypress:dw:qa:serverless:run"
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/edr_workflows/mki_security_solution_defend_workflows.sh cypress:dw:qa:serverless:run
         key: test_defend_workflows
         agents:
@@ -14,7 +14,7 @@ steps:
           localSsdInterface: nvme
           machineType: n2-standard-4
         timeout_in_minutes: 300
-        parallelism: 6
+        parallelism: 5
         retry:
           automatic:
             - exit_status: "*"
@@ -91,7 +91,7 @@ steps:
 #            - exit_status: "1"
 #              limit: 1
 
-      - label: "Running edr_workflows:policy_response:qa:serverless"
+      - label: "API - DW - Running edr_workflows:policy_response:qa:serverless"
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh edr_workflows:policy_response:qa:serverless
         key: edr_workflows:policy_response:qa:serverless
         agents:
@@ -108,7 +108,7 @@ steps:
             - exit_status: "1"
               limit: 1
 
-      - label: "Running edr_workflows:resolver:qa:serverless"
+      - label: "API - DW - Running edr_workflows:resolver:qa:serverless"
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh edr_workflows:resolver:qa:serverless
         key: edr_workflows:resolver:qa:serverless
         agents:
@@ -125,7 +125,7 @@ steps:
             - exit_status: "1"
               limit: 1
 
-      - label: "Running edr_workflows:response_actions:qa:serverless"
+      - label: "API - DW - Running edr_workflows:response_actions:qa:serverless"
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh edr_workflows:response_actions:qa:serverless
         key: edr_workflows:response_actions:qa:serverless
         agents:
@@ -141,3 +141,24 @@ steps:
           automatic:
             - exit_status: "1"
               limit: 1
+
+  - group: "Osquery MKI - Defend Workflows"
+    key: cypress_test_osquery_defend_workflows
+    steps:
+      - label: "Osquery - Cypress - DW - Running cypress:qa:serverless:run"
+        command: .buildkite/scripts/pipelines/security_solution_quality_gate/edr_workflows/mki_security_solution_defend_workflows_osquery.sh cypress:qa:serverless:run
+        key: test_osquery_defend_workflows
+        agents:
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-prod
+          provider: gcp
+          enableNestedVirtualization: true
+          localSsds: 1
+          localSsdInterface: nvme
+          machineType: n2-standard-4
+        timeout_in_minutes: 300
+        parallelism: 3
+        retry:
+          automatic:
+            - exit_status: "*"
+              limit: 1

.buildkite/scripts/common/util.sh

@@ -12,7 +12,7 @@ is_pr_with_label() {
 
   IFS=',' read -ra labels <<< "${GITHUB_PR_LABELS:-}"
 
-  for label in "${labels[@]}"
+  for label in "${labels[@]:-}"
   do
     if [ "$label" == "$match" ]; then
       return

.buildkite/scripts/pipelines/security_solution_quality_gate/edr_workflows/mki_security_solution_defend_workflows_osquery.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -euo pipefail
+
+if [ -z "$1" ]
+  then
+    echo "No target script from the package.json file, is supplied"
+    exit 1
+fi
+
+
+source .buildkite/scripts/common/util.sh
+.buildkite/scripts/bootstrap.sh
+
+export JOB=kibana-defend-workflows-osquery-serverless-cypress
+
+buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" "true"
+
+source .buildkite/scripts/pipelines/security_solution_quality_gate/prepare_vault_entries.sh
+
+cd x-pack/plugins/osquery
+set +e
+
+export BK_ANALYTICS_API_KEY=$(vault_get security-solution-quality-gate serverless-cypress-defend-workflows)
+
+echo "--- Running the tests for target $1"
+BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn $1; status=$?; yarn junit:merge || :; exit $status

.buildkite/scripts/steps/artifacts/docker_image.sh

@@ -118,6 +118,12 @@ echo "--- Trigger image tag update"
 if [[ "$BUILDKITE_BRANCH" == "$KIBANA_BASE_BRANCH" ]] && [[ "${BUILDKITE_PULL_REQUEST:-false}" == "false" ]]; then
   cat << EOF | buildkite-agent pipeline upload
 steps:
+  - label: "Trigger cve-slo-status pipeline for $KIBANA_IMAGE"
+    trigger: cve-slo-status
+    build:
+      env:
+        CONTAINER: "$KIBANA_IMAGE"
+    soft_fail: true
   - label: ":argo: Update kibana image tag for kibana-controller using gpctl"
     branches: main
     trigger: gpctl-promote-with-e2e-tests

.buildkite/scripts/steps/capture_oas_snapshot.sh

@@ -5,7 +5,7 @@ set -euo pipefail
 source .buildkite/scripts/common/util.sh
 
 echo --- Capture OAS snapshot
-cmd="node scripts/capture_oas_snapshot --include-path /api/status"
+cmd="node scripts/capture_oas_snapshot --include-path /api/status --include-path /api/alerting/rule/ --include-path /api/alerting/rules"
 if is_pr && ! is_auto_commit_disabled; then
   cmd="$cmd --update"
 fi

.buildkite/scripts/steps/checks.sh

@@ -15,3 +15,4 @@ fi
 .buildkite/scripts/steps/code_generation/security_solution_codegen.sh
 .buildkite/scripts/steps/openapi_bundling/security_solution_openapi_bundling.sh
 .buildkite/scripts/steps/code_generation/osquery_codegen.sh
+.buildkite/scripts/steps/openapi_bundling/final_merge.sh

.buildkite/scripts/steps/checks/event_log.sh

@@ -8,7 +8,25 @@ echo --- Check Event Log Schema
 
 # event log schema is pinned to a specific version of ECS
 ECS_STABLE_VERSION=1.8
-git clone --depth 1 -b $ECS_STABLE_VERSION https://github.com/elastic/ecs.git ../ecs
+
+# we can potentially skip this check on a local env, if ../ecs is present, and modified by the developer
+if [[ "${CI:-false}" =~ ^(0|false)$ ]] && [[ -d '../ecs' ]]; then
+  LOCAL_ECS_BRANCH=$(git -C ../ecs branch --show-current)
+  if [[ "$LOCAL_ECS_BRANCH" != "$ECS_STABLE_VERSION" ]]; then
+    echo "Skipping event log schema check because ECS schema is not on $ECS_STABLE_VERSION."
+    exit 0
+  fi
+
+  TOUCHED_FILES=$(git -C ../ecs status --porcelain)
+  if [[ -n "$TOUCHED_FILES" ]]; then
+    echo "Skipping event log schema check because ECS schema files have been modified."
+    exit 0
+  fi
+
+  echo "../ecs is already cloned and @ $ECS_STABLE_VERSION"
+else
+  git clone --depth 1 -b $ECS_STABLE_VERSION https://github.com/elastic/ecs.git ../ecs
+fi
 
 node x-pack/plugins/event_log/scripts/create_schemas.js