Merge branch 'main' into flaky-tests-2024-02-06

.buildkite/pipelines/security_solution/security_solution_cypress.yml

@@ -1,96 +1,122 @@
 steps:
-  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:explore
-    label: 'Serverless MKI QA Explore - Security Solution Cypress Tests'
+  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/create_periodic_test_docker_image.sh
+    label: Build kibana image
+    key: build_image
     agents:
-      queue: n2-4-spot
-    # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
-    timeout_in_minutes: 300
-    parallelism: 4
+      queue: n2-16-spot
+    timeout_in_minutes: 60
     retry:
       automatic:
-        - exit_status: '*'
-          limit: 1
+        - exit_status: "-1"
+          limit: 3
 
-  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:investigations
-    label: 'Serverless MKI QA Investigations - Security Solution Cypress Tests'
+  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/upload_image_metadata.sh
+    label: "Upload runtime info"
+    key: upload_runtime_info
+    depends_on: build_image
     agents:
       queue: n2-4-spot
-    # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
     timeout_in_minutes: 300
-    parallelism: 8
     retry:
       automatic:
-        - exit_status: '*'
+        - exit_status: "*"
           limit: 1
 
-  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:rule_management
-    label: 'Serverless MKI QA Rule Management - Security Solution Cypress Tests'
-    agents:
-      queue: n2-4-spot
-    # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
-    timeout_in_minutes: 300
-    parallelism: 8
-    retry:
-      automatic:
-        - exit_status: '*'
-          limit: 1
+  - group: "Execute Tests"
+    depends_on: build_image
+    steps:
+    # - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:explore
+    #   label: 'Serverless MKI QA Explore - Security Solution Cypress Tests'
+    #   agents:
+    #     queue: n2-4-spot
+    #   # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
+    #   timeout_in_minutes: 300
+    #   parallelism: 4
+    #   retry:
+    #     automatic:
+    #       - exit_status: '*'
+    #         limit: 1
 
-  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:rule_management:prebuilt_rules
-    label: 'Serverless MKI QA Rule Management - Prebuilt Rules - Security Solution Cypress Tests'
-    agents:
-      queue: n2-4-spot
-    # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
-    timeout_in_minutes: 300
-    parallelism: 2
-    retry:
-      automatic:
-        - exit_status: '*'
-          limit: 1
+    # - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:investigations
+    #   label: 'Serverless MKI QA Investigations - Security Solution Cypress Tests'
+    #   agents:
+    #     queue: n2-4-spot
+    #   # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
+    #   timeout_in_minutes: 300
+    #   parallelism: 8
+    #   retry:
+    #     automatic:
+    #       - exit_status: '*'
+    #         limit: 1
 
-  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:detection_engine
-    label: 'Serverless MKI QA Detection Engine - Security Solution Cypress Tests'
-    agents:
-      queue: n2-4-spot
-    # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
-    timeout_in_minutes: 300
-    parallelism: 8
-    retry:
-      automatic:
-        - exit_status: '*'
-          limit: 1
+    # - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:rule_management
+    #   label: 'Serverless MKI QA Rule Management - Security Solution Cypress Tests'
+    #   agents:
+    #     queue: n2-4-spot
+    #   # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
+    #   timeout_in_minutes: 300
+    #   parallelism: 8
+    #   retry:
+    #     automatic:
+    #       - exit_status: '*'
+    #         limit: 1
 
-  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:detection_engine:exceptions
-    label: 'Serverless MKI QA Detection Engine - Exceptions - Security Solution Cypress Tests'
-    agents:
-      queue: n2-4-spot
-    # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
-    timeout_in_minutes: 300
-    parallelism: 6
-    retry:
-      automatic:
-        - exit_status: '*'
-          limit: 1
+    # - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:rule_management:prebuilt_rules
+    #   label: 'Serverless MKI QA Rule Management - Prebuilt Rules - Security Solution Cypress Tests'
+    #   agents:
+    #     queue: n2-4-spot
+    #   # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
+    #   timeout_in_minutes: 300
+    #   parallelism: 2
+    #   retry:
+    #     automatic:
+    #       - exit_status: '*'
+    #         limit: 1
 
-  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:ai_assistant
-    label: 'Serverless MKI QA AI Assistant - Security Solution Cypress Tests'
-    agents:
-      queue: n2-4-spot
-    # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
-    timeout_in_minutes: 300
-    parallelism: 1
-    retry:
-      automatic:
-        - exit_status: '*'
-          limit: 1
+    # - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:detection_engine
+    #   label: 'Serverless MKI QA Detection Engine - Security Solution Cypress Tests'
+    #   agents:
+    #     queue: n2-4-spot
+    #   # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
+    #   timeout_in_minutes: 300
+    #   parallelism: 8
+    #   retry:
+    #     automatic:
+    #       - exit_status: '*'
+    #         limit: 1
 
-  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:entity_analytics
-    label: 'Serverless MKI QA Entity Analytics - Security Solution Cypress Tests'
-    agents:
-      queue: n2-4-spot
-    # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
-    timeout_in_minutes: 300
-    parallelism: 2
-    retry:
-      automatic:
-        - exit_status: '*'
-          limit: 1
+    # - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:detection_engine:exceptions
+    #   label: 'Serverless MKI QA Detection Engine - Exceptions - Security Solution Cypress Tests'
+    #   agents:
+    #     queue: n2-4-spot
+    #   # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
+    #   timeout_in_minutes: 300
+    #   parallelism: 6
+    #   retry:
+    #     automatic:
+    #       - exit_status: '*'
+    #         limit: 1
+
+    - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:ai_assistant
+      label: 'Serverless MKI QA AI Assistant - Security Solution Cypress Tests'
+      agents:
+        queue: n2-4-spot
+      # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
+      timeout_in_minutes: 300
+      parallelism: 1
+      retry:
+        automatic:
+          - exit_status: '*'
+            limit: 1
+
+    # - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:entity_analytics
+    #   label: 'Serverless MKI QA Entity Analytics - Security Solution Cypress Tests'
+    #   agents:
+    #     queue: n2-4-spot
+    #   # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
+    #   timeout_in_minutes: 300
+    #   parallelism: 2
+    #   retry:
+    #     automatic:
+    #       - exit_status: '*'
+    #         limit: 1

.buildkite/scripts/common/activate_service_account.sh

@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${BASH_SOURCE[0]}")/vault_fns.sh"
+
+BUCKET_OR_EMAIL="${1:-}"
+GCLOUD_EMAIL_POSTFIX="elastic-kibana-ci.iam.gserviceaccount.com"
+GCLOUD_SA_PROXY_EMAIL="kibana-ci-sa-proxy@$GCLOUD_EMAIL_POSTFIX"
+
+if [[ -z "$BUCKET_OR_EMAIL" ]]; then
+  echo "Usage: $0 <bucket_name|email>"
+  exit 1
+elif [[ "$BUCKET_OR_EMAIL" == "--unset-impersonation" ]]; then
+  echo "Unsetting impersonation"
+  gcloud config unset auth/impersonate_service_account
+  exit 0
+elif [[ "$BUCKET_OR_EMAIL" == "--logout-gcloud" ]]; then
+  echo "Logging out of gcloud"
+  if [[ -x "$(command -v gcloud)" ]] && [[ "$(gcloud auth list 2>/dev/null | grep $GCLOUD_SA_PROXY_EMAIL)" != "" ]]; then
+    gcloud auth revoke $GCLOUD_SA_PROXY_EMAIL --no-user-output-enabled
+  fi
+  exit 0
+fi
+
+CURRENT_GCLOUD_USER=$(gcloud auth list --filter="status=ACTIVE" --format="value(account)")
+
+# Verify that the service account proxy is activated
+if [[ "$CURRENT_GCLOUD_USER" != "$GCLOUD_SA_PROXY_EMAIL" ]]; then
+    if [[ -x "$(command -v gcloud)" ]]; then
+      if [[ -z "${KIBANA_SERVICE_ACCOUNT_PROXY_KEY:-}" ]]; then
+        echo "KIBANA_SERVICE_ACCOUNT_PROXY_KEY is not set, cannot activate service account $GCLOUD_SA_PROXY_EMAIL."
+        exit 1
+      fi
+
+      AUTH_RESULT=$(gcloud auth activate-service-account --key-file="$KIBANA_SERVICE_ACCOUNT_PROXY_KEY" || "FAILURE")
+      if [[ "$AUTH_RESULT" == "FAILURE" ]]; then
+        echo "Failed to activate service account $GCLOUD_SA_PROXY_EMAIL."
+        exit 1
+      else
+        echo "Activated service account $GCLOUD_SA_PROXY_EMAIL"
+      fi
+    else
+      echo "gcloud is not installed, cannot activate service account $GCLOUD_SA_PROXY_EMAIL."
+      exit 1
+    fi
+fi
+
+# Check if the arg is a service account e-mail or a bucket name
+EMAIL=""
+if [[ "$BUCKET_OR_EMAIL" =~ ^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$ ]]; then
+  EMAIL="$BUCKET_OR_EMAIL"
+elif [[ "$BUCKET_OR_EMAIL" =~ ^gs://* ]]; then
+  BUCKET_NAME="${BUCKET_OR_EMAIL:5}"
+else
+  BUCKET_NAME="$BUCKET_OR_EMAIL"
+fi
+
+if [[ -z "$EMAIL" ]]; then
+  case "$BUCKET_NAME" in
+    "elastic-kibana-coverage-live")
+      EMAIL="kibana-ci-access-coverage@$GCLOUD_EMAIL_POSTFIX"
+      ;;
+    "kibana-ci-es-snapshots-daily")
+      EMAIL="kibana-ci-access-es-snapshots@$GCLOUD_EMAIL_POSTFIX"
+      ;;
+    "kibana-so-types-snapshots")
+      EMAIL="kibana-ci-access-so-snapshots@$GCLOUD_EMAIL_POSTFIX"
+      ;;
+    "kibana-performance")
+      EMAIL="kibana-ci-access-perf-stats@$GCLOUD_EMAIL_POSTFIX"
+      ;;
+    "ci-artifacts.kibana.dev")
+      EMAIL="kibana-ci-access-artifacts@$GCLOUD_EMAIL_POSTFIX"
+      ;;
+    *)
+      EMAIL="$BUCKET_NAME@$GCLOUD_EMAIL_POSTFIX"
+      ;;
+  esac
+fi
+
+# Activate the service account
+echo "Impersonating $EMAIL"
+gcloud config set auth/impersonate_service_account "$EMAIL"
+echo "Activated service account $EMAIL"

.buildkite/scripts/common/setup_bazel.sh

@@ -2,6 +2,8 @@
 
 source .buildkite/scripts/common/util.sh
 
+echo '--- Setting up bazel'
+
 echo "[bazel] writing .bazelrc"
 cat <<EOF > $KIBANA_DIR/.bazelrc
   # Generated by .buildkite/scripts/common/setup_bazel.sh
@@ -27,16 +29,16 @@ if [[ "$BAZEL_CACHE_MODE" == "gcs" ]]; then
 
   echo "[bazel] using GCS bucket: $BAZEL_BUCKET"
 
-cat <<EOF >> $KIBANA_DIR/.bazelrc
+  cat <<EOF >> $KIBANA_DIR/.bazelrc
   build --remote_cache=https://storage.googleapis.com/$BAZEL_BUCKET
-  build --google_default_credentials
+  build --google_credentials=$BAZEL_REMOTE_CACHE_CREDENTIALS_FILE
 EOF
 fi
 
 if [[ "$BAZEL_CACHE_MODE" == "populate-local-gcs" ]]; then
   echo "[bazel] enabling caching with GCS buckets for local dev"
 
-cat <<EOF >> $KIBANA_DIR/.bazelrc
+  cat <<EOF >> $KIBANA_DIR/.bazelrc
   build --remote_cache=https://storage.googleapis.com/kibana-local-bazel-remote-cache
   build --google_credentials=$BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE
 EOF

.buildkite/scripts/common/util.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+source "$(dirname "${BASH_SOURCE[0]}")/vault_fns.sh"
+
 is_pr() {
   [[ "${GITHUB_PR_NUMBER-}" ]] && return
   false
@@ -170,48 +172,3 @@ npm_install_global() {
 download_artifact() {
   retry 3 1 timeout 3m buildkite-agent artifact download "$@"
 }
-
-# TODO: remove after https://github.com/elastic/kibana-operations/issues/15 is done
-if [[ "${VAULT_ADDR:-}" == *"secrets.elastic.co"* ]]; then
-  VAULT_PATH_PREFIX="secret/kibana-issues/dev"
-  VAULT_KV_PREFIX="secret/kibana-issues/dev"
-  IS_LEGACY_VAULT_ADDR=true
-else
-  VAULT_PATH_PREFIX="secret/ci/elastic-kibana"
-  VAULT_KV_PREFIX="kv/ci-shared/kibana-deployments"
-  IS_LEGACY_VAULT_ADDR=false
-fi
-export IS_LEGACY_VAULT_ADDR
-
-vault_get() {
-  key_path=$1
-  field=$2
-
-  fullPath="$VAULT_PATH_PREFIX/$key_path"
-
-  if [[ -z "${2:-}" || "${2:-}" =~ ^-.* ]]; then
-    retry 5 5 vault read "$fullPath" "${@:2}"
-  else
-    retry 5 5 vault read -field="$field" "$fullPath" "${@:3}"
-  fi
-}
-
-vault_set() {
-  key_path=$1
-  shift
-  fields=("$@")
-
-
-  fullPath="$VAULT_PATH_PREFIX/$key_path"
-
-  # shellcheck disable=SC2068
-  retry 5 5 vault write "$fullPath" ${fields[@]}
-}
-
-vault_kv_set() {
-  kv_path=$1
-  shift
-  fields=("$@")
-
-  vault kv put "$VAULT_KV_PREFIX/$kv_path" "${fields[@]}"
-}