From 015b910de2e9048c63eb2c7cbaa83dcdbcddd4c4 Mon Sep 17 00:00:00 2001 From: Alex Szabo <alex.szabo@elastic.co> Date: Wed, 6 Sep 2023 13:22:43 +0200 Subject: [PATCH] [Ops] Run kibana quality gate suites (#165346) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Sets up quality gate triggering in Kibana. The tests are mostly triggering external pipelines and relying on their results. Here's an example run: https://buildkite.com/elastic/kibana-tests/builds/28#job-018a69a6-c860-405e-ab2b-bce2aed07df3 According to [this doc](https://docs.google.com/document/d/15rx2Z-soL20An0nBUcXX0o_HHf1OU_IgrHXgz20NndI/edit) many of the quality gates are really required for the QA->Staging promotion step. Most of the tests are in the QA stage: - [fleet smoke tests](https://buildkite.com/elastic/fleet-smoke-tests) - ~~[QAF RAC load tests](https://buildkite.com/elastic/appex-qa-rac-alert-load)~~ Removed, see https://github.com/elastic/kibana/pull/165346#discussion_r1316822164 - [QAF serverless tests](https://buildkite.com/elastic/appex-qa-kibana-serverless-ftr-tests) - [control plane QA smoke tests](https://buildkite.com/elastic/ess-k8s-qa-e2e-tests-daily) - [security solution tests](.buildkite/scripts/pipelines/security_solution_quality_gate/pipeline.sh) + manual check for confirming manual tests - Manual confirmation 👍 Staging has: - [control plane staging smoke tests](https://buildkite.com/elastic/ess-k8s-staging-e2e-tests) - Manual confirmation 👍 Production has: - [control plane production smoke tests](https://buildkite.com/elastic/ess-k8s-production-e2e-tests) - Manual confirmation 👍 ### Quirks - ~~`SKIP_KIBANA_HOOKS=1` needs to be set from the triggering job~~ Split into https://github.com/elastic/kibana/pull/165597 - The pipeline can only be tested from the non-fork, `elastic/kibana` repo's branches (because buildkite doesn't see forks' branches) - Soft fails added, to not block the release pipeline in case we still need to adjust/work on some unstable tests Reference: https://docs.google.com/document/d/15rx2Z-soL20An0nBUcXX0o_HHf1OU_IgrHXgz20NndI/edit Depends on: #165009 Closes: https://github.com/elastic/kibana-operations/issues/10 --------- Co-authored-by: Tiago Costa <tiago.costa@elastic.co> Co-authored-by: Thomas Watson <watson@elastic.co> --- .../quality-gates/pipeline.kibana-tests.yaml | 2 +- .../pipeline.tests-production.yaml | 19 ++----- .../quality-gates/pipeline.tests-qa.yaml | 49 ++++++++++++------- .../quality-gates/pipeline.tests-staging.yaml | 17 ++----- 4 files changed, 40 insertions(+), 47 deletions(-) diff --git a/.buildkite/pipelines/quality-gates/pipeline.kibana-tests.yaml b/.buildkite/pipelines/quality-gates/pipeline.kibana-tests.yaml index 0acdb66f8d5f2..467df501bc9ca 100644 --- a/.buildkite/pipelines/quality-gates/pipeline.kibana-tests.yaml +++ b/.buildkite/pipelines/quality-gates/pipeline.kibana-tests.yaml @@ -18,7 +18,7 @@ steps: - label: ":pipeline::grey_question::seedling: Trigger Kibana Tests for ${ENVIRONMENT}" env: QG_PIPELINE_LOCATION: ".buildkite/pipelines/quality-gates" - command: "make -C /agent run-environment-tests" + command: "make -C /agent run-environment-tests" # will trigger https://buildkite.com/elastic/kibana-tests agents: image: "docker.elastic.co/ci-agent-images/quality-gate-seedling:0.0.2" diff --git a/.buildkite/pipelines/quality-gates/pipeline.tests-production.yaml b/.buildkite/pipelines/quality-gates/pipeline.tests-production.yaml index 1c30a7f734df4..32878e2fc09cd 100644 --- a/.buildkite/pipelines/quality-gates/pipeline.tests-production.yaml +++ b/.buildkite/pipelines/quality-gates/pipeline.tests-production.yaml @@ -3,27 +3,18 @@ # A failure in this pipeline build will prevent further progression to the subsequent stage. steps: - - label: ":pipeline::fleet::seedling: Trigger Observability Kibana Tests for ${ENVIRONMENT}" - command: echo "replace me with Observability specific Kibana tests" - agents: - image: "docker.elastic.co/ci-agent-images/basic-buildkite-agent:1688566364" - - - label: ":pipeline::lock::seedling: Trigger Security Kibana Tests for ${ENVIRONMENT}" - command: echo "replace me with Security specific Kibana tests" - agents: - image: "docker.elastic.co/ci-agent-images/basic-buildkite-agent:1688566364" - - - label: ":rocket: Run cp e2e tests" - trigger: "ess-k8s-production-e2e-tests" + - label: ":pipeline::rocket::seedling: Trigger control-plane e2e tests" + trigger: "ess-k8s-production-e2e-tests" # https://buildkite.com/elastic/ess-k8s-production-e2e-tests build: - message: "${BUILDKITE_MESSAGE}" env: REGION_ID: aws-us-east-1 - NAME_PREFIX: ci_test_${SERVICE}-promotion_ + NAME_PREFIX: ci_test_kibana-promotion_ + message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-production.yaml)" - wait: ~ - label: ":judge::seedling: Trigger Manual Tests Phase" command: "make -C /agent trigger-manual-verification-phase" + if: build.branch == "main" agents: image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.2" diff --git a/.buildkite/pipelines/quality-gates/pipeline.tests-qa.yaml b/.buildkite/pipelines/quality-gates/pipeline.tests-qa.yaml index e03e986f65833..e7fbf640cf565 100644 --- a/.buildkite/pipelines/quality-gates/pipeline.tests-qa.yaml +++ b/.buildkite/pipelines/quality-gates/pipeline.tests-qa.yaml @@ -3,37 +3,48 @@ # this pipeline build will prevent further progression to the subsequent stage. steps: - - label: ":pipeline::kibana::seedling: Trigger Kibana Tests for ${ENVIRONMENT}" - command: echo "replace me with Kibana specific tests" - agents: - image: "docker.elastic.co/ci-agent-images/basic-buildkite-agent:1688566364" + - label: ":pipeline::kibana::seedling: Trigger Kibana Serverless Tests for ${ENVIRONMENT}" + trigger: appex-qa-kibana-serverless-ftr-tests # https://buildkite.com/elastic/appex-qa-kibana-serverless-ftr-tests + soft_fail: true # Remove this before release or when tests stabilize + build: + env: + ENVIRONMENT: ${ENVIRONMENT} + message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-qa.yaml)" - - label: ":pipeline::fleet::seedling: Trigger Fleet Kibana Tests for ${ENVIRONMENT}" - command: echo "replace me with Fleet specific Kibana tests" - agents: - image: "docker.elastic.co/ci-agent-images/basic-buildkite-agent:1688566364" + - group: ":female-detective: Security Solution Tests" + key: "security" + steps: + - label: ":pipeline::female-detective::seedling: Trigger Security Solution quality gate script" + command: .buildkite/scripts/pipelines/security_solution_quality_gate/pipeline.sh - - label: ":pipeline::lock::seedling: Trigger Security Kibana Tests for ${ENVIRONMENT}" - command: echo "replace me with Security specific Kibana tests" - agents: - image: "docker.elastic.co/ci-agent-images/basic-buildkite-agent:1688566364" + - label: ":judge::seedling: Trigger Manual Tests Phase" + command: "make -C /agent trigger-manual-verification-phase" + if: build.branch == "main" + env: + TEAM_CHANNEL: "#kibana-mission-control" + agents: + image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.2" - - label: ":pipeline::lock::seedling: Trigger Control Plane Kibana Tests for ${ENVIRONMENT}" - command: echo "replace me with Control Plane specific Kibana tests" - agents: - image: "docker.elastic.co/ci-agent-images/basic-buildkite-agent:1688566364" + - label: ":pipeline::ship::seedling: Trigger Fleet serverless smoke tests for ${ENVIRONMENT}" + trigger: fleet-smoke-tests # https://buildkite.com/elastic/fleet-smoke-tests + soft_fail: true # Remove this before release + build: + env: + ENVIRONMENT: ${ENVIRONMENT} + message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-qa.yaml)" - - label: ":rocket: Run cp e2e tests" - trigger: "ess-k8s-qa-e2e-tests-daily" + - label: ":pipeline::rocket::seedling: Trigger control-plane e2e tests" + trigger: "ess-k8s-qa-e2e-tests-daily" # https://buildkite.com/elastic/ess-k8s-qa-e2e-tests-daily build: - message: "${BUILDKITE_MESSAGE}" env: REGION_ID: aws-eu-west-1 NAME_PREFIX: ci_test_kibana-promotion_ + message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-qa.yaml)" - wait: ~ - label: ":judge::seedling: Trigger Manual Tests Phase" command: "make -C /agent trigger-manual-verification-phase" + if: build.branch == "main" agents: image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.2" diff --git a/.buildkite/pipelines/quality-gates/pipeline.tests-staging.yaml b/.buildkite/pipelines/quality-gates/pipeline.tests-staging.yaml index 83bfd0d27e34c..a376ff2ff1884 100644 --- a/.buildkite/pipelines/quality-gates/pipeline.tests-staging.yaml +++ b/.buildkite/pipelines/quality-gates/pipeline.tests-staging.yaml @@ -3,27 +3,18 @@ # this pipeline build will prevent further progression to the subsequent stage. steps: - - label: ":pipeline::fleet::seedling: Trigger Observability Kibana Tests for ${ENVIRONMENT}" - command: echo "replace me with Observability specific Kibana tests" - agents: - image: "docker.elastic.co/ci-agent-images/basic-buildkite-agent:1688566364" - - - label: ":pipeline::lock::seedling: Trigger Security Kibana Tests for ${ENVIRONMENT}" - command: echo "replace me with Security specific Kibana tests" - agents: - image: "docker.elastic.co/ci-agent-images/basic-buildkite-agent:1688566364" - - - label: ":rocket: Run cp e2e tests" - trigger: "ess-k8s-staging-e2e-tests" + - label: ":pipeline::rocket::seedling: Trigger control-plane e2e tests" + trigger: "ess-k8s-staging-e2e-tests" # https://buildkite.com/elastic/ess-k8s-staging-e2e-tests build: - message: "${BUILDKITE_MESSAGE}" env: REGION_ID: aws-us-east-1 NAME_PREFIX: ci_test_kibana-promotion_ + message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-staging.yaml)" - wait: ~ - label: ":judge::seedling: Trigger Manual Tests Phase" command: "make -C /agent trigger-manual-verification-phase" + if: build.branch == "main" agents: image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.2"