Allow for disabling specific tests in scenarios

This commit adds several environment variables that allow more granular control over which tests are run by an integration test scenario. Signed-off-by: Christopher A. Snapp <[email protected]>
chef · Feb 18, 2020 · 32baadf · 32baadf
1 parent 492aea3
commit 32baadf
Show file tree

Hide file tree

Showing 36 changed files with 1,320 additions and 195 deletions.
diff --git a/.expeditor/integration_test.pipeline.sh b/.expeditor/integration_test.pipeline.sh
@@ -147,6 +147,15 @@ apply () {
   [[ -z "$TF_VAR_platform" ]] && TF_VAR_platform="${PLATFORM:-ubuntu-18.04}"
   export TF_VAR_platform
 
+  # apply optional test overrides if necessary
+  [[ -n "$ENABLE_SMOKE_TEST" ]] && export TF_VAR_enable_smoke_test="$ENABLE_SMOKE_TEST"
+  [[ -n "$ENABLE_PEDANT_TEST" ]] && export TF_VAR_enable_pedant_test="$ENABLE_PEDANT_TEST"
+  [[ -n "$ENABLE_PSQL_TEST" ]] && export TF_VAR_enable_psql_test="$ENABLE_PSQL_TEST"
+  [[ -n "$ENABLE_GATHER_LOGS_TEST" ]] && export TF_VAR_enable_gather_logs_test="$ENABLE_GATHER_LOGS_TEST"
+  [[ -n "$ENABLE_ADDON_PUSH_JOBS" ]] && export TF_VAR_enable_addon_push_jobs="$ENABLE_ADDON_PUSH_JOBS"
+  [[ -n "$ENABLE_ADDON_CHEF_MANAGE" ]] && export TF_VAR_enable_addon_chef_manage="$ENABLE_ADDON_CHEF_MANAGE"
+  [[ -n "$ENABLE_CHEF_BACKEND_DEMOTION" ]] && export TF_VAR_enable_chef_backend_demotion="$ENABLE_CHEF_BACKEND_DEMOTION"
+
   # setup the terraform workspace
   setup "$TF_VAR_scenario" "$TF_VAR_enable_ipv6" "${TF_VAR_platform}"
 

diff --git a/.expeditor/integration_test.pipeline.yml b/.expeditor/integration_test.pipeline.yml
@@ -12,6 +12,7 @@ steps:
       SCENARIO: omnibus-chef-backend
       PLATFORM: rhel-6
       ENABLE_IPV6: false
+      ENABLE_CHEF_BACKEND_DEMOTION: false
     expeditor:
       accounts:
         - aws/chef-cd
@@ -24,6 +25,7 @@ steps:
       SCENARIO: omnibus-chef-backend
       PLATFORM: rhel-7
       ENABLE_IPV6: false
+      ENABLE_CHEF_BACKEND_DEMOTION: false
     expeditor:
       accounts:
         - aws/chef-cd
@@ -36,6 +38,7 @@ steps:
       SCENARIO: omnibus-chef-backend
       PLATFORM: rhel-8
       ENABLE_IPV6: false
+      ENABLE_CHEF_BACKEND_DEMOTION: false
     expeditor:
       accounts:
         - aws/chef-cd
@@ -48,6 +51,7 @@ steps:
       SCENARIO: omnibus-chef-backend
       PLATFORM: ubuntu-16.04
       ENABLE_IPV6: false
+      ENABLE_CHEF_BACKEND_DEMOTION: false
     expeditor:
       accounts:
         - aws/chef-cd
@@ -60,6 +64,7 @@ steps:
       SCENARIO: omnibus-chef-backend
       PLATFORM: ubuntu-18.04
       ENABLE_IPV6: false
+      ENABLE_CHEF_BACKEND_DEMOTION: false
     expeditor:
       accounts:
         - aws/chef-cd
@@ -72,6 +77,7 @@ steps:
       SCENARIO: omnibus-external-openldap
       PLATFORM: rhel-6
       ENABLE_IPV6: false
+      ENABLE_PEDANT_TEST: false
     expeditor:
       accounts:
         - aws/chef-cd
@@ -84,6 +90,7 @@ steps:
       SCENARIO: omnibus-external-openldap
       PLATFORM: rhel-7
       ENABLE_IPV6: false
+      ENABLE_PEDANT_TEST: false
     expeditor:
       accounts:
         - aws/chef-cd
@@ -96,6 +103,7 @@ steps:
       SCENARIO: omnibus-external-openldap
       PLATFORM: rhel-8
       ENABLE_IPV6: false
+      ENABLE_PEDANT_TEST: false
     expeditor:
       accounts:
         - aws/chef-cd
@@ -108,6 +116,7 @@ steps:
       SCENARIO: omnibus-external-openldap
       PLATFORM: ubuntu-16.04
       ENABLE_IPV6: false
+      ENABLE_PEDANT_TEST: false
     expeditor:
       accounts:
         - aws/chef-cd
@@ -120,6 +129,7 @@ steps:
       SCENARIO: omnibus-external-openldap
       PLATFORM: ubuntu-18.04
       ENABLE_IPV6: false
+      ENABLE_PEDANT_TEST: false
     expeditor:
       accounts:
         - aws/chef-cd
@@ -168,6 +178,7 @@ steps:
       SCENARIO: omnibus-external-postgresql
       PLATFORM: rhel-6
       ENABLE_IPV6: false
+      ENABLE_ADDON_PUSH_JOBS: false
     expeditor:
       accounts:
         - aws/chef-cd
@@ -180,6 +191,7 @@ steps:
       SCENARIO: omnibus-external-postgresql
       PLATFORM: rhel-7
       ENABLE_IPV6: false
+      ENABLE_ADDON_PUSH_JOBS: false
     expeditor:
       accounts:
         - aws/chef-cd
@@ -192,6 +204,7 @@ steps:
       SCENARIO: omnibus-external-postgresql
       PLATFORM: rhel-8
       ENABLE_IPV6: false
+      ENABLE_ADDON_PUSH_JOBS: false
     expeditor:
       accounts:
         - aws/chef-cd
@@ -204,6 +217,7 @@ steps:
       SCENARIO: omnibus-external-postgresql
       PLATFORM: ubuntu-16.04
       ENABLE_IPV6: false
+      ENABLE_ADDON_PUSH_JOBS: false
     expeditor:
       accounts:
         - aws/chef-cd
@@ -216,6 +230,7 @@ steps:
       SCENARIO: omnibus-external-postgresql
       PLATFORM: ubuntu-18.04
       ENABLE_IPV6: false
+      ENABLE_ADDON_PUSH_JOBS: false
     expeditor:
       accounts:
         - aws/chef-cd
@@ -444,6 +459,7 @@ steps:
       SCENARIO: omnibus-chef-backend
       PLATFORM: rhel-6
       ENABLE_IPV6: true
+      ENABLE_CHEF_BACKEND_DEMOTION: false
     expeditor:
       accounts:
         - aws/chef-cd
@@ -456,6 +472,7 @@ steps:
       SCENARIO: omnibus-chef-backend
       PLATFORM: rhel-7
       ENABLE_IPV6: true
+      ENABLE_CHEF_BACKEND_DEMOTION: false
     expeditor:
       accounts:
         - aws/chef-cd
@@ -468,6 +485,7 @@ steps:
       SCENARIO: omnibus-chef-backend
       PLATFORM: rhel-8
       ENABLE_IPV6: true
+      ENABLE_CHEF_BACKEND_DEMOTION: false
     expeditor:
       accounts:
         - aws/chef-cd
@@ -480,6 +498,7 @@ steps:
       SCENARIO: omnibus-chef-backend
       PLATFORM: ubuntu-16.04
       ENABLE_IPV6: true
+      ENABLE_CHEF_BACKEND_DEMOTION: false
     expeditor:
       accounts:
         - aws/chef-cd
@@ -492,6 +511,7 @@ steps:
       SCENARIO: omnibus-chef-backend
       PLATFORM: ubuntu-18.04
       ENABLE_IPV6: true
+      ENABLE_CHEF_BACKEND_DEMOTION: false
     expeditor:
       accounts:
         - aws/chef-cd
@@ -504,6 +524,7 @@ steps:
       SCENARIO: omnibus-external-openldap
       PLATFORM: rhel-6
       ENABLE_IPV6: true
+      ENABLE_PEDANT_TEST: false
     expeditor:
       accounts:
         - aws/chef-cd
@@ -516,6 +537,7 @@ steps:
       SCENARIO: omnibus-external-openldap
       PLATFORM: rhel-7
       ENABLE_IPV6: true
+      ENABLE_PEDANT_TEST: false
     expeditor:
       accounts:
         - aws/chef-cd
@@ -528,6 +550,7 @@ steps:
       SCENARIO: omnibus-external-openldap
       PLATFORM: rhel-8
       ENABLE_IPV6: true
+      ENABLE_PEDANT_TEST: false
     expeditor:
       accounts:
         - aws/chef-cd
@@ -540,6 +563,7 @@ steps:
       SCENARIO: omnibus-external-openldap
       PLATFORM: ubuntu-16.04
       ENABLE_IPV6: true
+      ENABLE_PEDANT_TEST: false
     expeditor:
       accounts:
         - aws/chef-cd
@@ -552,6 +576,7 @@ steps:
       SCENARIO: omnibus-external-openldap
       PLATFORM: ubuntu-18.04
       ENABLE_IPV6: true
+      ENABLE_PEDANT_TEST: false
     expeditor:
       accounts:
         - aws/chef-cd
@@ -600,6 +625,7 @@ steps:
       SCENARIO: omnibus-external-postgresql
       PLATFORM: rhel-6
       ENABLE_IPV6: true
+      ENABLE_ADDON_PUSH_JOBS: false
     expeditor:
       accounts:
         - aws/chef-cd
@@ -612,6 +638,7 @@ steps:
       SCENARIO: omnibus-external-postgresql
       PLATFORM: rhel-7
       ENABLE_IPV6: true
+      ENABLE_ADDON_PUSH_JOBS: false
     expeditor:
       accounts:
         - aws/chef-cd
@@ -624,6 +651,7 @@ steps:
       SCENARIO: omnibus-external-postgresql
       PLATFORM: rhel-8
       ENABLE_IPV6: true
+      ENABLE_ADDON_PUSH_JOBS: false
     expeditor:
       accounts:
         - aws/chef-cd
@@ -636,6 +664,7 @@ steps:
       SCENARIO: omnibus-external-postgresql
       PLATFORM: ubuntu-16.04
       ENABLE_IPV6: true
+      ENABLE_ADDON_PUSH_JOBS: false
     expeditor:
       accounts:
         - aws/chef-cd
@@ -648,6 +677,7 @@ steps:
       SCENARIO: omnibus-external-postgresql
       PLATFORM: ubuntu-18.04
       ENABLE_IPV6: true
+      ENABLE_ADDON_PUSH_JOBS: false
     expeditor:
       accounts:
         - aws/chef-cd

diff --git a/terraform/aws/Makefile b/terraform/aws/Makefile
@@ -58,6 +58,34 @@ else
 	export TF_VAR_enable_ipv6 ?= "true"
 endif
 
+ifneq ($(ENABLE_SMOKE_TEST),)
+    export TF_VAR_enable_smoke_test ?= $(ENABLE_SMOKE_TEST)
+endif
+
+ifneq ($(ENABLE_PEDANT_TEST),)
+    export TF_VAR_enable_pedant_test ?= $(ENABLE_PEDANT_TEST)
+endif
+
+ifneq ($(ENABLE_PSQL_TEST),)
+    export TF_VAR_enable_psql_test ?= $(ENABLE_PSQL_TEST)
+endif
+
+ifneq ($(ENABLE_GATHER_LOGS_TEST),)
+    export TF_VAR_enable_gather_logs_test ?= $(ENABLE_GATHER_LOGS_TEST)
+endif
+
+ifneq ($(ENABLE_ADDON_PUSH_JOBS),)
+    export TF_VAR_enable_addon_push_jobs ?= $(ENABLE_ADDON_PUSH_JOBS)
+endif
+
+ifneq ($(ENABLE_ADDON_CHEF_MANAGE),)
+    export TF_VAR_enable_addon_chef_manage ?= $(ENABLE_ADDON_CHEF_MANAGE)
+endif
+
+ifneq ($(ENABLE_CHEF_BACKEND_DEMOTION),)
+    export TF_VAR_enable_chef_backend_demotion ?= $(ENABLE_CHEF_BACKEND_DEMOTION)
+endif
+
 ifneq ($(SCENARIO),)
 	export TF_VAR_scenario ?= $(SCENARIO)
 endif

diff --git a/terraform/aws/README.md b/terraform/aws/README.md
@@ -56,6 +56,13 @@ Environment variables are used to control how the scenarios are executed and can
 | `AWS_DEFAULT_INSTANCE_TYPE` | The AWS instance type that determines the amount of resources server instances are allocated. | t2.medium (default) |
 | `PLATFORM` | The operating system used by server instances. | rhel-6, rhel-7, rhel-8, ubuntu-16.04, ubuntu-18.04, sles-12 |
 | `ENABLE_IPV6` | Use IPv6 in the chef-server.rb config and /etc/hosts | true (default) |
+| `ENABLE_SMOKE_TEST` | Enable Chef Infra Server smoke test. | true (default) |
+| `ENABLE_PEDANT_TEST` | Enable full Chef Infra Server pedant test. | true (default) |
+| `ENABLE_PSQL_TEST` | Enable testing of Chef Infra Server PostgreSQL database. | true (default) |
+| `ENABLE_GATHER_LOGS_TEST` | Enable testing of Chef Infra Server gathering logs. | true (default) |
+| `ENABLE_ADDON_PUSH_JOBS` | Enable testing of Push Jobs addon. | true (default) |
+| `ENABLE_ADDON_CHEF_MANAGE` | Enable testing of Chef Manage addon. | true (default) |
+| `ENABLE_CHEF_BACKEND_DEMOTION` | Enable testing of chef-backend leadership demotion. | true (default) |
 
 ### Scenario Lifecycle
 

diff --git a/terraform/aws/scenarios/omnibus-chef-backend/files/test_chef_backend-demotion.sh b/terraform/aws/scenarios/omnibus-chef-backend/files/test_chef_backend-demotion.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -evx
+
+if [ "${ENABLE_CHEF_BACKEND_DEMOTION:-true}" = 'true' ]; then
+    ret=0
+
+    echo -e '\nBEGIN TEST CHEF BACKEND DEMOTION\n'
+
+    echo 'Original Chef Backend leader status'
+    sudo chef-backend-ctl cluster-status | tee /tmp/chef-backend-status.orig
+
+    echo 'Demoting current leader'
+    sudo chef-backend-ctl demote >/dev/null 2>&1
+    sleep 30
+
+    echo 'Current Chef Backend leader status'
+    sudo chef-backend-ctl cluster-status | tee /tmp/chef-backend-status.cur
+
+    leaders=$(sudo awk '/leader.*leader/{print $3}' /tmp/chef-backend-status.{orig,cur} | sort -u | wc -l)
+
+    if [[ $leaders -ne 2 ]]; then
+        ret=1
+
+        echo 'ERROR: Leadership transfer was not successful'
+
+        sudo diff /tmp/chef-backend-status.{orig,cur}
+    fi
+
+    echo -e '\nEND TEST CHEF BACKEND DEMOTION\n'
+
+    exit $ret
+else
+    echo -e '\n**SKIP** TEST CHEF BACKEND DEMOTION **SKIP**\n'
+fi