Bump minikube, helm, and test kubernetes version (#1051)

* Bump kubernetes version used in test

* Fix test failures with kubernetes version bump

* Use older version of minikube that worksaround permissions issue on
nginx

* Update TestRemoteChartInstall to use bitnami nginx chart instead of deprecated chartmuseum

* Service name is release name with nginx

* Fix helm install tests from k8s upgrade

* Restore full testing for a full regression check

* Bump helm to latest

.circleci/config.yml

@@ -1,17 +1,17 @@
 env: &env
   environment:
     GRUNTWORK_INSTALLER_VERSION: v0.0.36
-    MODULE_CI_VERSION: v0.33.3
+    MODULE_CI_VERSION: v0.41.1
     MODULE_GCP_CI_VERSION: v0.1.1
     TERRAFORM_VERSION: 1.0.3
     PACKER_VERSION: 1.7.4
     TERRAGRUNT_VERSION: v0.32.3
     OPA_VERSION: v0.33.1
     GO_VERSION: 1.16.3
     GO111MODULE: auto
-    K8S_VERSION: v1.15.0  # Same as EKS
-    MINIKUBE_VERSION: v1.9.2
-    HELM_VERSION: v3.1.1
+    K8S_VERSION: v1.18.20  # Same as EKS
+    MINIKUBE_VERSION: v1.22.0
+    HELM_VERSION: v3.8.0
     KUBECONFIG: /home/circleci/.kube/config
     BIN_BUILD_PARALLELISM: 3
 
@@ -25,11 +25,14 @@ defaults: &defaults
 minikube_defaults: &minikube_defaults
   machine:
     enabled: true
-    image: "ubuntu-1604:201903-01"
+    image: ubuntu-1604:202104-01
   <<: *env
 
 setup_minikube: &setup_minikube
-  command: setup-minikube --k8s-version "$K8S_VERSION" --minikube-version "$MINIKUBE_VERSION"
+  command: |
+    sudo apt update -y
+    sudo apt install -y conntrack
+    setup-minikube --k8s-version "$K8S_VERSION" --minikube-version "$MINIKUBE_VERSION"
 
 install_helm: &install_helm
   name: install helm
@@ -333,7 +336,7 @@ workflows:
             tags:
               only: /^v.*/
 
-      - test:
+      - kubernetes_test:
           context:
             - Gruntwork Admin
           requires:
@@ -342,16 +345,16 @@ workflows:
             tags:
               only: /^v.*/
 
-      - gcp_test:
+      - helm_test:
           context:
-            - Gruntwork GCP
+            - Gruntwork Admin
           requires:
             - setup
           filters:
             tags:
               only: /^v.*/
 
-      - kubernetes_test:
+      - test:
           context:
             - Gruntwork Admin
           requires:
@@ -360,9 +363,9 @@ workflows:
             tags:
               only: /^v.*/
 
-      - helm_test:
+      - gcp_test:
           context:
-            - Gruntwork Admin
+            - Gruntwork GCP
           requires:
             - setup
           filters:

modules/helm/install_test.go

@@ -1,3 +1,4 @@
+//go:build kubeall || helm
 // +build kubeall helm
 
 // NOTE: we have build tags to differentiate kubernetes tests from non-kubernetes tests, and further differentiate helm
@@ -22,7 +23,13 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-// Test that we can install a remote chart (e.g stable/chartmuseum)
+const (
+	remoteChartSource  = "https://charts.bitnami.com/bitnami"
+	remoteChartName    = "nginx"
+	remoteChartVersion = "9.7.4"
+)
+
+// Test that we can install a remote chart (e.g bitnami/nginx)
 func TestRemoteChartInstall(t *testing.T) {
 	t.Parallel()
 
@@ -49,13 +56,13 @@ func TestRemoteChartInstall(t *testing.T) {
 	// Add the stable repo under a random name so as not to touch existing repo configs
 	uniqueName := strings.ToLower(fmt.Sprintf("terratest-%s", random.UniqueId()))
 	defer RemoveRepo(t, options, uniqueName)
-	AddRepo(t, options, uniqueName, "https://charts.helm.sh/stable")
-	helmChart := fmt.Sprintf("%s/chartmuseum", uniqueName)
+	AddRepo(t, options, uniqueName, remoteChartSource)
+	helmChart := fmt.Sprintf("%s/%s", uniqueName, remoteChartName)
 
 	// Generate a unique release name so we can defer the delete before installing
 	releaseName := fmt.Sprintf(
-		"chartmuseum-%s",
-		strings.ToLower(random.UniqueId()),
+		"%s-%s",
+		remoteChartName, strings.ToLower(random.UniqueId()),
 	)
 	defer Delete(t, options, releaseName, true)
 
@@ -64,19 +71,18 @@ func TestRemoteChartInstall(t *testing.T) {
 	require.Error(t, InstallE(t, options, helmChart, releaseName))
 
 	// Fix chart version and retry install
-	options.Version = "2.3.0"
+	options.Version = remoteChartVersion
 	// Test that passing extra arguments doesn't error, by changing default timeout
 	options.ExtraArgs = map[string][]string{"install": []string{"--timeout", "5m1s"}}
 	options.ExtraArgs["delete"] = []string{"--timeout", "5m1s"}
 	require.NoError(t, InstallE(t, options, helmChart, releaseName))
 	waitForRemoteChartPods(t, kubectlOptions, releaseName, 1)
 
 	// Verify service is accessible. Wait for it to become available and then hit the endpoint.
-	// Service name is RELEASE_NAME-CHART_NAME
-	serviceName := fmt.Sprintf("%s-chartmuseum", releaseName)
+	serviceName := releaseName
 	k8s.WaitUntilServiceAvailable(t, kubectlOptions, serviceName, 10, 1*time.Second)
 	service := k8s.GetService(t, kubectlOptions, serviceName)
-	endpoint := k8s.GetServiceEndpoint(t, kubectlOptions, service, 8080)
+	endpoint := k8s.GetServiceEndpoint(t, kubectlOptions, service, 80)
 
 	// Setup a TLS configuration to submit with the helper, a blank struct is acceptable
 	tlsConfig := tls.Config{}
@@ -97,7 +103,10 @@ func waitForRemoteChartPods(t *testing.T, kubectlOptions *k8s.KubectlOptions, re
 	// Get pod and wait for it to be avaialable
 	// To get the pod, we need to filter it using the labels that the helm chart creates
 	filters := metav1.ListOptions{
-		LabelSelector: fmt.Sprintf("app=chartmuseum,release=%s", releaseName),
+		LabelSelector: fmt.Sprintf(
+			"app.kubernetes.io/name=%s,app.kubernetes.io/instance=%s",
+			remoteChartName, releaseName,
+		),
 	}
 	k8s.WaitUntilNumPodsCreated(t, kubectlOptions, filters, podCount, 30, 10*time.Second)
 	pods := k8s.ListPods(t, kubectlOptions, filters)

modules/helm/upgrade_test.go

@@ -1,3 +1,4 @@
+//go:build kubeall || helm
 // +build kubeall helm
 
 // NOTE: we have build tags to differentiate kubernetes tests from non-kubernetes tests, and further differentiate helm
@@ -46,20 +47,20 @@ func TestRemoteChartInstallUpgradeRollback(t *testing.T) {
 	// Add the stable repo under a random name so as not to touch existing repo configs
 	uniqueName := strings.ToLower(fmt.Sprintf("terratest-%s", random.UniqueId()))
 	defer RemoveRepo(t, options, uniqueName)
-	AddRepo(t, options, uniqueName, "https://charts.helm.sh/stable")
-	helmChart := fmt.Sprintf("%s/chartmuseum", uniqueName)
+	AddRepo(t, options, uniqueName, remoteChartSource)
+	helmChart := fmt.Sprintf("%s/%s", uniqueName, remoteChartName)
 
 	// Generate a unique release name so we can defer the delete before installing
 	releaseName := fmt.Sprintf(
-		"chartmuseum-%s",
-		strings.ToLower(random.UniqueId()),
+		"%s-%s",
+		remoteChartName, strings.ToLower(random.UniqueId()),
 	)
 	defer Delete(t, options, releaseName, true)
 	Install(t, options, helmChart, releaseName)
 	waitForRemoteChartPods(t, kubectlOptions, releaseName, 1)
 
 	// Setting replica count to 2 to check the upgrade functionality.
-	// After successful upgrade , the count of pods should be equal to 2.
+	// After successful upgrade, the count of pods should be equal to 2.
 	options.SetValues = map[string]string{
 		"replicaCount": "2",
 		"service.type": "NodePort",
@@ -71,11 +72,10 @@ func TestRemoteChartInstallUpgradeRollback(t *testing.T) {
 	waitForRemoteChartPods(t, kubectlOptions, releaseName, 2)
 
 	// Verify service is accessible. Wait for it to become available and then hit the endpoint.
-	// Service name is RELEASE_NAME-CHART_NAME
-	serviceName := fmt.Sprintf("%s-chartmuseum", releaseName)
+	serviceName := releaseName
 	k8s.WaitUntilServiceAvailable(t, kubectlOptions, serviceName, 10, 1*time.Second)
 	service := k8s.GetService(t, kubectlOptions, serviceName)
-	endpoint := k8s.GetServiceEndpoint(t, kubectlOptions, service, 8080)
+	endpoint := k8s.GetServiceEndpoint(t, kubectlOptions, service, 80)
 
 	// Setup a TLS configuration to submit with the helper, a blank struct is acceptable
 	tlsConfig := tls.Config{}

modules/k8s/ingress_test.go

@@ -1,3 +1,4 @@
+//go:build kubernetes
 // +build kubernetes
 
 // NOTE: we have build tags to differentiate kubernetes tests from non-kubernetes tests. This is done because minikube
@@ -35,7 +36,7 @@ func TestGetIngressEReturnsCorrectIngressInCorrectNamespace(t *testing.T) {
 
 	uniqueID := strings.ToLower(random.UniqueId())
 	options := NewKubectlOptions("", "", uniqueID)
-	configData := fmt.Sprintf(EXAMPLE_INGRESS_DEPLOYMENT_YAML_TEMPLATE, uniqueID, uniqueID, uniqueID, uniqueID)
+	configData := fmt.Sprintf(exampleIngressDeploymentYamlTemplate, uniqueID, uniqueID, uniqueID, uniqueID, uniqueID)
 	KubectlApplyFromString(t, options, configData)
 	defer KubectlDeleteFromString(t, options, configData)
 
@@ -49,7 +50,7 @@ func TestListIngressesReturnsCorrectIngressInCorrectNamespace(t *testing.T) {
 
 	uniqueID := strings.ToLower(random.UniqueId())
 	options := NewKubectlOptions("", "", uniqueID)
-	configData := fmt.Sprintf(EXAMPLE_INGRESS_DEPLOYMENT_YAML_TEMPLATE, uniqueID, uniqueID, uniqueID, uniqueID)
+	configData := fmt.Sprintf(exampleIngressDeploymentYamlTemplate, uniqueID, uniqueID, uniqueID, uniqueID, uniqueID)
 	KubectlApplyFromString(t, options, configData)
 	defer KubectlDeleteFromString(t, options, configData)
 
@@ -66,14 +67,14 @@ func TestWaitUntilIngressAvailableReturnsSuccessfully(t *testing.T) {
 
 	uniqueID := strings.ToLower(random.UniqueId())
 	options := NewKubectlOptions("", "", uniqueID)
-	configData := fmt.Sprintf(EXAMPLE_INGRESS_DEPLOYMENT_YAML_TEMPLATE, uniqueID, uniqueID, uniqueID, uniqueID)
+	configData := fmt.Sprintf(exampleIngressDeploymentYamlTemplate, uniqueID, uniqueID, uniqueID, uniqueID, uniqueID)
 	KubectlApplyFromString(t, options, configData)
 	defer KubectlDeleteFromString(t, options, configData)
 
 	WaitUntilIngressAvailableV1Beta1(t, options, ExampleIngressName, 60, 5*time.Second)
 }
 
-const EXAMPLE_INGRESS_DEPLOYMENT_YAML_TEMPLATE = `---
+const exampleIngressDeploymentYamlTemplate = `---
 apiVersion: v1
 kind: Namespace
 metadata:
@@ -123,7 +124,7 @@ spec:
   rules:
   - http:
       paths:
-      - path: /app
+      - path: /app-%s
         backend:
           serviceName: nginx-service
           servicePort: 80