From 5e833db183453171749187719505eeaac81d2af3 Mon Sep 17 00:00:00 2001 From: Yoriyasu Yano <430092+yorinasub17@users.noreply.github.com> Date: Thu, 27 Jan 2022 14:00:03 -0600 Subject: [PATCH] Bump minikube, helm, and test kubernetes version (#1051) * Bump kubernetes version used in test * Fix test failures with kubernetes version bump * Use older version of minikube that worksaround permissions issue on nginx * Update TestRemoteChartInstall to use bitnami nginx chart instead of deprecated chartmuseum * Service name is release name with nginx * Fix helm install tests from k8s upgrade * Restore full testing for a full regression check * Bump helm to latest --- .circleci/config.yml | 27 +++++++++++++++------------ modules/helm/install_test.go | 29 +++++++++++++++++++---------- modules/helm/upgrade_test.go | 16 ++++++++-------- modules/k8s/ingress_test.go | 11 ++++++----- 4 files changed, 48 insertions(+), 35 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 92aa18bbd..8681d4b74 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,7 +1,7 @@ env: &env environment: GRUNTWORK_INSTALLER_VERSION: v0.0.36 - MODULE_CI_VERSION: v0.33.3 + MODULE_CI_VERSION: v0.41.1 MODULE_GCP_CI_VERSION: v0.1.1 TERRAFORM_VERSION: 1.0.3 PACKER_VERSION: 1.7.4 @@ -9,9 +9,9 @@ env: &env OPA_VERSION: v0.33.1 GO_VERSION: 1.16.3 GO111MODULE: auto - K8S_VERSION: v1.15.0 # Same as EKS - MINIKUBE_VERSION: v1.9.2 - HELM_VERSION: v3.1.1 + K8S_VERSION: v1.18.20 # Same as EKS + MINIKUBE_VERSION: v1.22.0 + HELM_VERSION: v3.8.0 KUBECONFIG: /home/circleci/.kube/config BIN_BUILD_PARALLELISM: 3 @@ -25,11 +25,14 @@ defaults: &defaults minikube_defaults: &minikube_defaults machine: enabled: true - image: "ubuntu-1604:201903-01" + image: ubuntu-1604:202104-01 <<: *env setup_minikube: &setup_minikube - command: setup-minikube --k8s-version "$K8S_VERSION" --minikube-version "$MINIKUBE_VERSION" + command: | + sudo apt update -y + sudo apt install -y conntrack + setup-minikube --k8s-version "$K8S_VERSION" --minikube-version "$MINIKUBE_VERSION" install_helm: &install_helm name: install helm @@ -333,7 +336,7 @@ workflows: tags: only: /^v.*/ - - test: + - kubernetes_test: context: - Gruntwork Admin requires: @@ -342,16 +345,16 @@ workflows: tags: only: /^v.*/ - - gcp_test: + - helm_test: context: - - Gruntwork GCP + - Gruntwork Admin requires: - setup filters: tags: only: /^v.*/ - - kubernetes_test: + - test: context: - Gruntwork Admin requires: @@ -360,9 +363,9 @@ workflows: tags: only: /^v.*/ - - helm_test: + - gcp_test: context: - - Gruntwork Admin + - Gruntwork GCP requires: - setup filters: diff --git a/modules/helm/install_test.go b/modules/helm/install_test.go index a67a5c787..b5d7ce8a5 100644 --- a/modules/helm/install_test.go +++ b/modules/helm/install_test.go @@ -1,3 +1,4 @@ +//go:build kubeall || helm // +build kubeall helm // NOTE: we have build tags to differentiate kubernetes tests from non-kubernetes tests, and further differentiate helm @@ -22,7 +23,13 @@ import ( "github.com/stretchr/testify/require" ) -// Test that we can install a remote chart (e.g stable/chartmuseum) +const ( + remoteChartSource = "https://charts.bitnami.com/bitnami" + remoteChartName = "nginx" + remoteChartVersion = "9.7.4" +) + +// Test that we can install a remote chart (e.g bitnami/nginx) func TestRemoteChartInstall(t *testing.T) { t.Parallel() @@ -49,13 +56,13 @@ func TestRemoteChartInstall(t *testing.T) { // Add the stable repo under a random name so as not to touch existing repo configs uniqueName := strings.ToLower(fmt.Sprintf("terratest-%s", random.UniqueId())) defer RemoveRepo(t, options, uniqueName) - AddRepo(t, options, uniqueName, "https://charts.helm.sh/stable") - helmChart := fmt.Sprintf("%s/chartmuseum", uniqueName) + AddRepo(t, options, uniqueName, remoteChartSource) + helmChart := fmt.Sprintf("%s/%s", uniqueName, remoteChartName) // Generate a unique release name so we can defer the delete before installing releaseName := fmt.Sprintf( - "chartmuseum-%s", - strings.ToLower(random.UniqueId()), + "%s-%s", + remoteChartName, strings.ToLower(random.UniqueId()), ) defer Delete(t, options, releaseName, true) @@ -64,7 +71,7 @@ func TestRemoteChartInstall(t *testing.T) { require.Error(t, InstallE(t, options, helmChart, releaseName)) // Fix chart version and retry install - options.Version = "2.3.0" + options.Version = remoteChartVersion // Test that passing extra arguments doesn't error, by changing default timeout options.ExtraArgs = map[string][]string{"install": []string{"--timeout", "5m1s"}} options.ExtraArgs["delete"] = []string{"--timeout", "5m1s"} @@ -72,11 +79,10 @@ func TestRemoteChartInstall(t *testing.T) { waitForRemoteChartPods(t, kubectlOptions, releaseName, 1) // Verify service is accessible. Wait for it to become available and then hit the endpoint. - // Service name is RELEASE_NAME-CHART_NAME - serviceName := fmt.Sprintf("%s-chartmuseum", releaseName) + serviceName := releaseName k8s.WaitUntilServiceAvailable(t, kubectlOptions, serviceName, 10, 1*time.Second) service := k8s.GetService(t, kubectlOptions, serviceName) - endpoint := k8s.GetServiceEndpoint(t, kubectlOptions, service, 8080) + endpoint := k8s.GetServiceEndpoint(t, kubectlOptions, service, 80) // Setup a TLS configuration to submit with the helper, a blank struct is acceptable tlsConfig := tls.Config{} @@ -97,7 +103,10 @@ func waitForRemoteChartPods(t *testing.T, kubectlOptions *k8s.KubectlOptions, re // Get pod and wait for it to be avaialable // To get the pod, we need to filter it using the labels that the helm chart creates filters := metav1.ListOptions{ - LabelSelector: fmt.Sprintf("app=chartmuseum,release=%s", releaseName), + LabelSelector: fmt.Sprintf( + "app.kubernetes.io/name=%s,app.kubernetes.io/instance=%s", + remoteChartName, releaseName, + ), } k8s.WaitUntilNumPodsCreated(t, kubectlOptions, filters, podCount, 30, 10*time.Second) pods := k8s.ListPods(t, kubectlOptions, filters) diff --git a/modules/helm/upgrade_test.go b/modules/helm/upgrade_test.go index 1f55a73ec..afef4f7c4 100644 --- a/modules/helm/upgrade_test.go +++ b/modules/helm/upgrade_test.go @@ -1,3 +1,4 @@ +//go:build kubeall || helm // +build kubeall helm // NOTE: we have build tags to differentiate kubernetes tests from non-kubernetes tests, and further differentiate helm @@ -46,20 +47,20 @@ func TestRemoteChartInstallUpgradeRollback(t *testing.T) { // Add the stable repo under a random name so as not to touch existing repo configs uniqueName := strings.ToLower(fmt.Sprintf("terratest-%s", random.UniqueId())) defer RemoveRepo(t, options, uniqueName) - AddRepo(t, options, uniqueName, "https://charts.helm.sh/stable") - helmChart := fmt.Sprintf("%s/chartmuseum", uniqueName) + AddRepo(t, options, uniqueName, remoteChartSource) + helmChart := fmt.Sprintf("%s/%s", uniqueName, remoteChartName) // Generate a unique release name so we can defer the delete before installing releaseName := fmt.Sprintf( - "chartmuseum-%s", - strings.ToLower(random.UniqueId()), + "%s-%s", + remoteChartName, strings.ToLower(random.UniqueId()), ) defer Delete(t, options, releaseName, true) Install(t, options, helmChart, releaseName) waitForRemoteChartPods(t, kubectlOptions, releaseName, 1) // Setting replica count to 2 to check the upgrade functionality. - // After successful upgrade , the count of pods should be equal to 2. + // After successful upgrade, the count of pods should be equal to 2. options.SetValues = map[string]string{ "replicaCount": "2", "service.type": "NodePort", @@ -71,11 +72,10 @@ func TestRemoteChartInstallUpgradeRollback(t *testing.T) { waitForRemoteChartPods(t, kubectlOptions, releaseName, 2) // Verify service is accessible. Wait for it to become available and then hit the endpoint. - // Service name is RELEASE_NAME-CHART_NAME - serviceName := fmt.Sprintf("%s-chartmuseum", releaseName) + serviceName := releaseName k8s.WaitUntilServiceAvailable(t, kubectlOptions, serviceName, 10, 1*time.Second) service := k8s.GetService(t, kubectlOptions, serviceName) - endpoint := k8s.GetServiceEndpoint(t, kubectlOptions, service, 8080) + endpoint := k8s.GetServiceEndpoint(t, kubectlOptions, service, 80) // Setup a TLS configuration to submit with the helper, a blank struct is acceptable tlsConfig := tls.Config{} diff --git a/modules/k8s/ingress_test.go b/modules/k8s/ingress_test.go index a74ac0da7..26e0b9af9 100644 --- a/modules/k8s/ingress_test.go +++ b/modules/k8s/ingress_test.go @@ -1,3 +1,4 @@ +//go:build kubernetes // +build kubernetes // NOTE: we have build tags to differentiate kubernetes tests from non-kubernetes tests. This is done because minikube @@ -35,7 +36,7 @@ func TestGetIngressEReturnsCorrectIngressInCorrectNamespace(t *testing.T) { uniqueID := strings.ToLower(random.UniqueId()) options := NewKubectlOptions("", "", uniqueID) - configData := fmt.Sprintf(EXAMPLE_INGRESS_DEPLOYMENT_YAML_TEMPLATE, uniqueID, uniqueID, uniqueID, uniqueID) + configData := fmt.Sprintf(exampleIngressDeploymentYamlTemplate, uniqueID, uniqueID, uniqueID, uniqueID, uniqueID) KubectlApplyFromString(t, options, configData) defer KubectlDeleteFromString(t, options, configData) @@ -49,7 +50,7 @@ func TestListIngressesReturnsCorrectIngressInCorrectNamespace(t *testing.T) { uniqueID := strings.ToLower(random.UniqueId()) options := NewKubectlOptions("", "", uniqueID) - configData := fmt.Sprintf(EXAMPLE_INGRESS_DEPLOYMENT_YAML_TEMPLATE, uniqueID, uniqueID, uniqueID, uniqueID) + configData := fmt.Sprintf(exampleIngressDeploymentYamlTemplate, uniqueID, uniqueID, uniqueID, uniqueID, uniqueID) KubectlApplyFromString(t, options, configData) defer KubectlDeleteFromString(t, options, configData) @@ -66,14 +67,14 @@ func TestWaitUntilIngressAvailableReturnsSuccessfully(t *testing.T) { uniqueID := strings.ToLower(random.UniqueId()) options := NewKubectlOptions("", "", uniqueID) - configData := fmt.Sprintf(EXAMPLE_INGRESS_DEPLOYMENT_YAML_TEMPLATE, uniqueID, uniqueID, uniqueID, uniqueID) + configData := fmt.Sprintf(exampleIngressDeploymentYamlTemplate, uniqueID, uniqueID, uniqueID, uniqueID, uniqueID) KubectlApplyFromString(t, options, configData) defer KubectlDeleteFromString(t, options, configData) WaitUntilIngressAvailableV1Beta1(t, options, ExampleIngressName, 60, 5*time.Second) } -const EXAMPLE_INGRESS_DEPLOYMENT_YAML_TEMPLATE = `--- +const exampleIngressDeploymentYamlTemplate = `--- apiVersion: v1 kind: Namespace metadata: @@ -123,7 +124,7 @@ spec: rules: - http: paths: - - path: /app + - path: /app-%s backend: serviceName: nginx-service servicePort: 80