From 015b910de2e9048c63eb2c7cbaa83dcdbcddd4c4 Mon Sep 17 00:00:00 2001
From: Alex Szabo <alex.szabo@elastic.co>
Date: Wed, 6 Sep 2023 13:22:43 +0200
Subject: [PATCH] [Ops] Run kibana quality gate suites (#165346)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

## Summary
Sets up quality gate triggering in Kibana. The tests are mostly
triggering external pipelines and relying on their results. Here's an
example run:
https://buildkite.com/elastic/kibana-tests/builds/28#job-018a69a6-c860-405e-ab2b-bce2aed07df3

According to [this
doc](https://docs.google.com/document/d/15rx2Z-soL20An0nBUcXX0o_HHf1OU_IgrHXgz20NndI/edit)
many of the quality gates are really required for the QA->Staging
promotion step.

Most of the tests are in the QA stage:
 - [fleet smoke tests](https://buildkite.com/elastic/fleet-smoke-tests)
- ~~[QAF RAC load
tests](https://buildkite.com/elastic/appex-qa-rac-alert-load)~~ Removed,
see https://github.com/elastic/kibana/pull/165346#discussion_r1316822164
- [QAF serverless
tests](https://buildkite.com/elastic/appex-qa-kibana-serverless-ftr-tests)
- [control plane QA smoke
tests](https://buildkite.com/elastic/ess-k8s-qa-e2e-tests-daily)
- [security solution
tests](.buildkite/scripts/pipelines/security_solution_quality_gate/pipeline.sh)
+ manual check for confirming manual tests
 - Manual confirmation 👍

Staging has:
- [control plane staging smoke
tests](https://buildkite.com/elastic/ess-k8s-staging-e2e-tests)
 - Manual confirmation 👍

Production has:
- [control plane production smoke
tests](https://buildkite.com/elastic/ess-k8s-production-e2e-tests)
 - Manual confirmation 👍

### Quirks
- ~~`SKIP_KIBANA_HOOKS=1` needs to be set from the triggering job~~
Split into https://github.com/elastic/kibana/pull/165597
- The pipeline can only be tested from the non-fork, `elastic/kibana`
repo's branches (because buildkite doesn't see forks' branches)
- Soft fails added, to not block the release pipeline in case we still
need to adjust/work on some unstable tests

Reference:
https://docs.google.com/document/d/15rx2Z-soL20An0nBUcXX0o_HHf1OU_IgrHXgz20NndI/edit
Depends on: #165009
Closes: https://github.com/elastic/kibana-operations/issues/10

---------

Co-authored-by: Tiago Costa <tiago.costa@elastic.co>
Co-authored-by: Thomas Watson <watson@elastic.co>
---
 .../quality-gates/pipeline.kibana-tests.yaml  |  2 +-
 .../pipeline.tests-production.yaml            | 19 ++-----
 .../quality-gates/pipeline.tests-qa.yaml      | 49 ++++++++++++-------
 .../quality-gates/pipeline.tests-staging.yaml | 17 ++-----
 4 files changed, 40 insertions(+), 47 deletions(-)

diff --git a/.buildkite/pipelines/quality-gates/pipeline.kibana-tests.yaml b/.buildkite/pipelines/quality-gates/pipeline.kibana-tests.yaml
index 0acdb66f8d5f2..467df501bc9ca 100644
--- a/.buildkite/pipelines/quality-gates/pipeline.kibana-tests.yaml
+++ b/.buildkite/pipelines/quality-gates/pipeline.kibana-tests.yaml
@@ -18,7 +18,7 @@ steps:
   - label: ":pipeline::grey_question::seedling: Trigger Kibana Tests for ${ENVIRONMENT}"
     env:
       QG_PIPELINE_LOCATION: ".buildkite/pipelines/quality-gates"
-    command: "make -C /agent run-environment-tests"
+    command: "make -C /agent run-environment-tests" # will trigger https://buildkite.com/elastic/kibana-tests
     agents:
       image: "docker.elastic.co/ci-agent-images/quality-gate-seedling:0.0.2"
 
diff --git a/.buildkite/pipelines/quality-gates/pipeline.tests-production.yaml b/.buildkite/pipelines/quality-gates/pipeline.tests-production.yaml
index 1c30a7f734df4..32878e2fc09cd 100644
--- a/.buildkite/pipelines/quality-gates/pipeline.tests-production.yaml
+++ b/.buildkite/pipelines/quality-gates/pipeline.tests-production.yaml
@@ -3,27 +3,18 @@
 # A failure in this pipeline build will prevent further progression to the subsequent stage.
 
 steps:
-  - label: ":pipeline::fleet::seedling: Trigger Observability Kibana Tests for ${ENVIRONMENT}"
-    command: echo "replace me with Observability specific Kibana tests"
-    agents:
-      image: "docker.elastic.co/ci-agent-images/basic-buildkite-agent:1688566364"
-
-  - label: ":pipeline::lock::seedling: Trigger Security Kibana Tests for ${ENVIRONMENT}"
-    command: echo "replace me with Security specific Kibana tests"
-    agents:
-      image: "docker.elastic.co/ci-agent-images/basic-buildkite-agent:1688566364"
-
-  - label: ":rocket: Run cp e2e tests"
-    trigger: "ess-k8s-production-e2e-tests"
+  - label: ":pipeline::rocket::seedling: Trigger control-plane e2e tests"
+    trigger: "ess-k8s-production-e2e-tests" # https://buildkite.com/elastic/ess-k8s-production-e2e-tests
     build:
-      message: "${BUILDKITE_MESSAGE}"
       env:
         REGION_ID: aws-us-east-1
-        NAME_PREFIX: ci_test_${SERVICE}-promotion_
+        NAME_PREFIX: ci_test_kibana-promotion_
+      message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-production.yaml)"
 
   - wait: ~
 
   - label: ":judge::seedling: Trigger Manual Tests Phase"
     command: "make -C /agent trigger-manual-verification-phase"
+    if: build.branch == "main"
     agents:
       image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.2"
diff --git a/.buildkite/pipelines/quality-gates/pipeline.tests-qa.yaml b/.buildkite/pipelines/quality-gates/pipeline.tests-qa.yaml
index e03e986f65833..e7fbf640cf565 100644
--- a/.buildkite/pipelines/quality-gates/pipeline.tests-qa.yaml
+++ b/.buildkite/pipelines/quality-gates/pipeline.tests-qa.yaml
@@ -3,37 +3,48 @@
 # this pipeline build will prevent further progression to the subsequent stage.
 
 steps:
-  - label: ":pipeline::kibana::seedling: Trigger Kibana Tests for ${ENVIRONMENT}"
-    command: echo "replace me with Kibana specific tests"
-    agents:
-      image: "docker.elastic.co/ci-agent-images/basic-buildkite-agent:1688566364"
+  - label: ":pipeline::kibana::seedling: Trigger Kibana Serverless Tests for ${ENVIRONMENT}"
+    trigger: appex-qa-kibana-serverless-ftr-tests # https://buildkite.com/elastic/appex-qa-kibana-serverless-ftr-tests
+    soft_fail: true # Remove this before release or when tests stabilize
+    build:
+      env:
+        ENVIRONMENT: ${ENVIRONMENT}
+      message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-qa.yaml)"
 
-  - label: ":pipeline::fleet::seedling: Trigger Fleet Kibana Tests for ${ENVIRONMENT}"
-    command: echo "replace me with Fleet specific Kibana tests"
-    agents:
-      image: "docker.elastic.co/ci-agent-images/basic-buildkite-agent:1688566364"
+  - group: ":female-detective: Security Solution Tests"
+    key: "security"
+    steps:
+      - label: ":pipeline::female-detective::seedling: Trigger Security Solution quality gate script"
+        command: .buildkite/scripts/pipelines/security_solution_quality_gate/pipeline.sh
 
-  - label: ":pipeline::lock::seedling: Trigger Security Kibana Tests for ${ENVIRONMENT}"
-    command: echo "replace me with Security specific Kibana tests"
-    agents:
-      image: "docker.elastic.co/ci-agent-images/basic-buildkite-agent:1688566364"
+      - label: ":judge::seedling: Trigger Manual Tests Phase"
+        command: "make -C /agent trigger-manual-verification-phase"
+        if: build.branch == "main"
+        env:
+          TEAM_CHANNEL: "#kibana-mission-control"
+        agents:
+          image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.2"
 
-  - label: ":pipeline::lock::seedling: Trigger Control Plane Kibana Tests for ${ENVIRONMENT}"
-    command: echo "replace me with Control Plane specific Kibana tests"
-    agents:
-      image: "docker.elastic.co/ci-agent-images/basic-buildkite-agent:1688566364"
+  - label: ":pipeline::ship::seedling: Trigger Fleet serverless smoke tests for ${ENVIRONMENT}"
+    trigger: fleet-smoke-tests # https://buildkite.com/elastic/fleet-smoke-tests
+    soft_fail: true # Remove this before release
+    build:
+      env:
+        ENVIRONMENT: ${ENVIRONMENT}
+      message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-qa.yaml)"
 
-  - label: ":rocket: Run cp e2e tests"
-    trigger: "ess-k8s-qa-e2e-tests-daily"
+  - label: ":pipeline::rocket::seedling: Trigger control-plane e2e tests"
+    trigger: "ess-k8s-qa-e2e-tests-daily" # https://buildkite.com/elastic/ess-k8s-qa-e2e-tests-daily
     build:
-      message: "${BUILDKITE_MESSAGE}"
       env:
         REGION_ID: aws-eu-west-1
         NAME_PREFIX: ci_test_kibana-promotion_
+      message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-qa.yaml)"
 
   - wait: ~
 
   - label: ":judge::seedling: Trigger Manual Tests Phase"
     command: "make -C /agent trigger-manual-verification-phase"
+    if: build.branch == "main"
     agents:
       image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.2"
diff --git a/.buildkite/pipelines/quality-gates/pipeline.tests-staging.yaml b/.buildkite/pipelines/quality-gates/pipeline.tests-staging.yaml
index 83bfd0d27e34c..a376ff2ff1884 100644
--- a/.buildkite/pipelines/quality-gates/pipeline.tests-staging.yaml
+++ b/.buildkite/pipelines/quality-gates/pipeline.tests-staging.yaml
@@ -3,27 +3,18 @@
 # this pipeline build will prevent further progression to the subsequent stage.
 
 steps:
-  - label: ":pipeline::fleet::seedling: Trigger Observability Kibana Tests for ${ENVIRONMENT}"
-    command: echo "replace me with Observability specific Kibana tests"
-    agents:
-      image: "docker.elastic.co/ci-agent-images/basic-buildkite-agent:1688566364"
-
-  - label: ":pipeline::lock::seedling: Trigger Security Kibana Tests for ${ENVIRONMENT}"
-    command: echo "replace me with Security specific Kibana tests"
-    agents:
-      image: "docker.elastic.co/ci-agent-images/basic-buildkite-agent:1688566364"
-
-  - label: ":rocket: Run cp e2e tests"
-    trigger: "ess-k8s-staging-e2e-tests"
+  - label: ":pipeline::rocket::seedling: Trigger control-plane e2e tests"
+    trigger: "ess-k8s-staging-e2e-tests" # https://buildkite.com/elastic/ess-k8s-staging-e2e-tests
     build:
-      message: "${BUILDKITE_MESSAGE}"
       env:
         REGION_ID: aws-us-east-1
         NAME_PREFIX: ci_test_kibana-promotion_
+      message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-staging.yaml)"
 
   - wait: ~
 
   - label: ":judge::seedling: Trigger Manual Tests Phase"
     command: "make -C /agent trigger-manual-verification-phase"
+    if: build.branch == "main"
     agents:
       image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.2"