From e6c9bc04fb4edb1b1488d4fa4d6cf9f6b8ca86b4 Mon Sep 17 00:00:00 2001 From: Rob Morgan Date: Tue, 14 May 2019 14:29:32 +0200 Subject: [PATCH 1/9] update kubergrunt to 0.3.9 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 350740c..8ad93de 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,7 +3,7 @@ defaults: &defaults environment: GRUNTWORK_INSTALLER_VERSION: v0.0.21 TERRATEST_LOG_PARSER_VERSION: v0.13.13 - KUBERGRUNT_VERSION: v0.3.8 + KUBERGRUNT_VERSION: v0.3.9 HELM_VERSION: v2.11.0 MODULE_CI_VERSION: v0.13.3 TERRAFORM_VERSION: 0.11.8 From d5a57ae838491e6aec9efe35cdcb478024cd4d67 Mon Sep 17 00:00:00 2001 From: Rob Morgan Date: Tue, 14 May 2019 14:30:52 +0200 Subject: [PATCH 2/9] update terratest to 0.14.6 --- test/Gopkg.lock | 14 ++++++++------ test/Gopkg.toml | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/test/Gopkg.lock b/test/Gopkg.lock index 8dbd30e..fe7d890 100644 --- a/test/Gopkg.lock +++ b/test/Gopkg.lock @@ -18,7 +18,7 @@ version = "v0.35.1" [[projects]] - digest = "1:f0ca0b3256cfe939f5b5d7fb9980d3805f0a9c0d7e2ce5c990112627e77018d7" + digest = "1:6fee3887aa799d9c2c2017d2738c5532b9cbee3fa94589232f11db00ff1e6ec3" name = "github.com/aws/aws-sdk-go" packages = [ "aws", @@ -60,6 +60,7 @@ "service/autoscaling", "service/cloudwatchlogs", "service/ec2", + "service/ecs", "service/iam", "service/kms", "service/rds", @@ -230,7 +231,7 @@ version = "v0.4.2" [[projects]] - digest = "1:12cf87df04f335d94d1346edd77536ca505a4dda0afd5137150cd440e5c2d32c" + digest = "1:9caaf503201ab0dfd178194b54306c45e72b6ea1dc7de5e432d933418149f9aa" name = "github.com/gruntwork-io/terratest" packages = [ "modules/aws", @@ -252,8 +253,8 @@ "modules/test-structure", ] pruneopts = "UT" - revision = "b6077c8c1ecfd290ee1f37e1e51472a34d30cb38" - version = "v0.13.29" + revision = "54393d85e00e2614efa6bd22e963fe174cc128d5" + version = "v0.14.6" [[projects]] digest = "1:a0cefd27d12712af4b5018dc7046f245e1e3b5760e2e848c30b171b570708f9b" @@ -496,7 +497,7 @@ [[projects]] branch = "master" - digest = "1:d586ebbd814cf4a680afbac35457f94a4ec9ae8305d9831b7a290c32baa0bb31" + digest = "1:a4df62df5ca682765882afdd14d4c904f2841a2fa455b044f4b04545f66cbd74" name = "google.golang.org/api" packages = [ "compute/v1", @@ -507,6 +508,7 @@ "internal", "iterator", "option", + "oslogin/v1", "storage/v1", "transport/http", "transport/http/internal/propagation", @@ -777,10 +779,10 @@ "github.com/gruntwork-io/terratest/modules/logger", "github.com/gruntwork-io/terratest/modules/random", "github.com/gruntwork-io/terratest/modules/retry", - "github.com/gruntwork-io/terratest/modules/shell", "github.com/gruntwork-io/terratest/modules/terraform", "github.com/gruntwork-io/terratest/modules/test-structure", "github.com/stretchr/testify/assert", + "github.com/stretchr/testify/require", ] solver-name = "gps-cdcl" solver-version = 1 diff --git a/test/Gopkg.toml b/test/Gopkg.toml index 2436a79..d96c92f 100644 --- a/test/Gopkg.toml +++ b/test/Gopkg.toml @@ -23,7 +23,7 @@ [[constraint]] name = "github.com/gruntwork-io/terratest" - version = "0.13.28" + version = "0.14.6" [prune] go-tests = true From 61a04f249b21de167d0b77f0ea0352b00e3a2ed3 Mon Sep 17 00:00:00 2001 From: Rob Morgan Date: Tue, 14 May 2019 14:44:02 +0200 Subject: [PATCH 3/9] start making the tests run in parallel again --- test/gke_basic_tiller_test.go | 30 ++++++++++++++++++------------ test/gke_cluster_test.go | 32 +++++++++++++++++++++----------- test/terratest_options.go | 16 ++++++++++------ test/test_helpers.go | 12 ++++++------ 4 files changed, 55 insertions(+), 35 deletions(-) diff --git a/test/gke_basic_tiller_test.go b/test/gke_basic_tiller_test.go index 3b47e20..ed3d5e7 100644 --- a/test/gke_basic_tiller_test.go +++ b/test/gke_basic_tiller_test.go @@ -2,6 +2,7 @@ package test import ( "fmt" + "os" "path/filepath" "strings" "testing" @@ -9,19 +10,17 @@ import ( "github.com/gruntwork-io/terratest/modules/gcp" "github.com/gruntwork-io/terratest/modules/helm" - "github.com/gruntwork-io/terratest/modules/http-helper" + http_helper "github.com/gruntwork-io/terratest/modules/http-helper" "github.com/gruntwork-io/terratest/modules/k8s" "github.com/gruntwork-io/terratest/modules/logger" "github.com/gruntwork-io/terratest/modules/random" "github.com/gruntwork-io/terratest/modules/terraform" - "github.com/gruntwork-io/terratest/modules/test-structure" + test_structure "github.com/gruntwork-io/terratest/modules/test-structure" + "github.com/stretchr/testify/require" ) func TestGKEBasicTiller(t *testing.T) { - // We are temporarily stopping the tests from running in parallel due to conflicting - // kubectl configs. This is a limitation in the current Terratest functions and will - // be fixed in a later release. - //t.Parallel() + t.Parallel() // Uncomment any of the following to skip that section during the test // os.Setenv("SKIP_create_test_copy_of_examples", "true") @@ -43,19 +42,27 @@ func TestGKEBasicTiller(t *testing.T) { test_structure.RunTestStage(t, "create_terratest_options", func() { gkeBasicTillerTerraformModulePath := test_structure.LoadString(t, workingDir, "gkeBasicTillerTerraformModulePath") + tmpKubeConfigPath := k8s.CopyHomeKubeConfigToTemp(t) + kubectlOptions := k8s.NewKubectlOptions("", tmpKubeConfigPath) uniqueID := random.UniqueId() project := gcp.GetGoogleProjectIDFromEnvVar(t) region := gcp.GetRandomRegion(t, project, nil, nil) - gkeClusterTerratestOptions := createGKEClusterTerraformOptions(t, uniqueID, project, region, gkeBasicTillerTerraformModulePath) + gkeClusterTerratestOptions := createGKEClusterTerraformOptions(t, uniqueID, project, region, + gkeBasicTillerTerraformModulePath, tmpKubeConfigPath) test_structure.SaveString(t, workingDir, "uniqueID", uniqueID) test_structure.SaveString(t, workingDir, "project", project) test_structure.SaveString(t, workingDir, "region", region) test_structure.SaveTerraformOptions(t, workingDir, gkeClusterTerratestOptions) + test_structure.SaveKubectlOptions(t, workingDir, kubectlOptions) }) defer test_structure.RunTestStage(t, "cleanup", func() { gkeClusterTerratestOptions := test_structure.LoadTerraformOptions(t, workingDir) terraform.Destroy(t, gkeClusterTerratestOptions) + + kubectlOptions := test_structure.LoadKubectlOptions(t, workingDir) + err := os.Remove(kubectlOptions.ConfigPath) + require.NoError(t, err) }) test_structure.RunTestStage(t, "terraform_apply", func() { @@ -64,18 +71,17 @@ func TestGKEBasicTiller(t *testing.T) { }) test_structure.RunTestStage(t, "wait_for_workers", func() { - verifyGkeNodesAreReady(t) + kubectlOptions := test_structure.LoadKubectlOptions(t, workingDir) + verifyGkeNodesAreReady(t, kubectlOptions) }) test_structure.RunTestStage(t, "helm_install", func() { // Path to the helm chart we will test helmChartPath := "charts/minimal-pod" - // Setup the kubectl config and context. Here we choose to use the defaults, which is: - // - HOME/.kube/config for the kubectl config file - // - Current context of the kubectl config file + // Load the temporary kubectl config file and use its current context // We also specify that we are working in the default namespace (required to get the Pod) - kubectlOptions := k8s.NewKubectlOptions("", "") + kubectlOptions := test_structure.LoadKubectlOptions(t, workingDir) kubectlOptions.Namespace = "default" // We generate a unique release name so that we can refer to after deployment. diff --git a/test/gke_cluster_test.go b/test/gke_cluster_test.go index f72ccb1..b75cd2d 100644 --- a/test/gke_cluster_test.go +++ b/test/gke_cluster_test.go @@ -1,22 +1,22 @@ package test import ( + "os" "path/filepath" "testing" "github.com/gruntwork-io/terratest/modules/gcp" + "github.com/gruntwork-io/terratest/modules/k8s" "github.com/gruntwork-io/terratest/modules/logger" "github.com/gruntwork-io/terratest/modules/random" "github.com/gruntwork-io/terratest/modules/shell" "github.com/gruntwork-io/terratest/modules/terraform" - "github.com/gruntwork-io/terratest/modules/test-structure" + test_structure "github.com/gruntwork-io/terratest/modules/test-structure" + "github.com/stretchr/testify/require" ) func TestGKECluster(t *testing.T) { - // We are temporarily stopping the tests from running in parallel due to conflicting - // kubectl configs. This is a limitation in the current Terratest functions and will - // be fixed in a later release. - //t.Parallel() + t.Parallel() var testcases = []struct { testName string @@ -38,10 +38,7 @@ func TestGKECluster(t *testing.T) { testCase := testCase t.Run(testCase.testName, func(t *testing.T) { - // We are temporarily stopping the tests from running in parallel due to conflicting - // kubectl configs. This is a limitation in the current Terratest functions and will - // be fixed in a later release. - //t.Parallel() + t.Parallel() // Uncomment any of the following to skip that section during the test //os.Setenv("SKIP_create_test_copy_of_examples", "true") @@ -63,19 +60,27 @@ func TestGKECluster(t *testing.T) { test_structure.RunTestStage(t, "create_terratest_options", func() { gkeClusterTerraformModulePath := test_structure.LoadString(t, workingDir, "gkeClusterTerraformModulePath") + tmpKubeConfigPath := k8s.CopyHomeKubeConfigToTemp(t) + kubectlOptions := k8s.NewKubectlOptions("", tmpKubeConfigPath) uniqueID := random.UniqueId() project := gcp.GetGoogleProjectIDFromEnvVar(t) region := gcp.GetRandomRegion(t, project, nil, nil) - gkeClusterTerratestOptions := createGKEClusterTerraformOptions(t, uniqueID, project, region, gkeClusterTerraformModulePath) + gkeClusterTerratestOptions := createGKEClusterTerraformOptions(t, uniqueID, project, region, + gkeClusterTerraformModulePath, tmpKubeConfigPath) test_structure.SaveString(t, workingDir, "uniqueID", uniqueID) test_structure.SaveString(t, workingDir, "project", project) test_structure.SaveString(t, workingDir, "region", region) test_structure.SaveTerraformOptions(t, workingDir, gkeClusterTerratestOptions) + test_structure.SaveKubectlOptions(t, workingDir, kubectlOptions) }) defer test_structure.RunTestStage(t, "cleanup", func() { gkeClusterTerratestOptions := test_structure.LoadTerraformOptions(t, workingDir) terraform.Destroy(t, gkeClusterTerratestOptions) + + kubectlOptions := test_structure.LoadKubectlOptions(t, workingDir) + err := os.Remove(kubectlOptions.ConfigPath) + require.NoError(t, err) }) test_structure.RunTestStage(t, "terraform_apply", func() { @@ -85,6 +90,7 @@ func TestGKECluster(t *testing.T) { test_structure.RunTestStage(t, "configure_kubectl", func() { gkeClusterTerratestOptions := test_structure.LoadTerraformOptions(t, workingDir) + kubectlOptions := test_structure.LoadKubectlOptions(t, workingDir) project := test_structure.LoadString(t, workingDir, "project") region := test_structure.LoadString(t, workingDir, "region") clusterName := gkeClusterTerratestOptions.Vars["cluster_name"].(string) @@ -93,13 +99,17 @@ func TestGKECluster(t *testing.T) { cmd := shell.Command{ Command: "gcloud", Args: []string{"beta", "container", "clusters", "get-credentials", clusterName, "--region", region, "--project", project}, + Env: map[string]string{ + "KUBECONFIG": kubectlOptions.ConfigPath, + }, } shell.RunCommand(t, cmd) }) test_structure.RunTestStage(t, "wait_for_workers", func() { - verifyGkeNodesAreReady(t) + kubectlOptions := test_structure.LoadKubectlOptions(t, workingDir) + verifyGkeNodesAreReady(t, kubectlOptions) }) }) } diff --git a/test/terratest_options.go b/test/terratest_options.go index 497ab8c..17d8d8c 100644 --- a/test/terratest_options.go +++ b/test/terratest_options.go @@ -14,14 +14,17 @@ func createGKEClusterTerraformOptions( project string, region string, templatePath string, + kubeConfigPath string, ) *terraform.Options { gkeClusterName := strings.ToLower(fmt.Sprintf("gke-cluster-%s", uniqueID)) + gkeServiceAccountName := strings.ToLower(fmt.Sprintf("gke-cluster-sa-%s", uniqueID)) terraformVars := map[string]interface{}{ - "region": region, - "location": region, - "project": project, - "cluster_name": gkeClusterName, + "region": region, + "location": region, + "project": project, + "cluster_name": gkeClusterName, + "cluster_service_account_name": gkeServiceAccountName, "tls_subject": map[string]string{ "common_name": "tiller", "org": "Gruntwork", @@ -30,8 +33,9 @@ func createGKEClusterTerraformOptions( "common_name": "helm", "org": "Gruntwork", }, - "force_undeploy": true, - "undeploy_release": true, + "force_undeploy": true, + "undeploy_release": true, + "kubectl_config_path": kubeConfigPath, } terratestOptions := terraform.Options{ diff --git a/test/test_helpers.go b/test/test_helpers.go index f2d51ed..33ef675 100644 --- a/test/test_helpers.go +++ b/test/test_helpers.go @@ -14,7 +14,7 @@ import ( // kubeWaitUntilNumNodes continuously polls the Kubernetes cluster until there are the expected number of nodes // registered (regardless of readiness). -func kubeWaitUntilNumNodes(t *testing.T, numNodes int, retries int, sleepBetweenRetries time.Duration) { +func kubeWaitUntilNumNodes(t *testing.T, kubectlOptions *k8s.KubectlOptions, numNodes int, retries int, sleepBetweenRetries time.Duration) { statusMsg := fmt.Sprintf("Wait for %d Kube Nodes to be registered.", numNodes) message, err := retry.DoWithRetryE( t, @@ -22,7 +22,7 @@ func kubeWaitUntilNumNodes(t *testing.T, numNodes int, retries int, sleepBetween retries, sleepBetweenRetries, func() (string, error) { - nodes, err := k8s.GetNodesE(t) + nodes, err := k8s.GetNodesE(t, kubectlOptions) if err != nil { return "", err } @@ -40,9 +40,9 @@ func kubeWaitUntilNumNodes(t *testing.T, numNodes int, retries int, sleepBetween } // Verify that all the nodes in the cluster reach the Ready state. -func verifyGkeNodesAreReady(t *testing.T) { - kubeWaitUntilNumNodes(t, 3, 30, 10*time.Second) - k8s.WaitUntilAllNodesReady(t, 30, 10*time.Second) - readyNodes := k8s.GetReadyNodes(t) +func verifyGkeNodesAreReady(t *testing.T, kubectlOptions *k8s.KubectlOptions) { + kubeWaitUntilNumNodes(t, kubectlOptions, 3, 30, 10*time.Second) + k8s.WaitUntilAllNodesReady(t, kubectlOptions, 30, 10*time.Second) + readyNodes := k8s.GetReadyNodes(t, kubectlOptions) assert.Equal(t, len(readyNodes), 3) } From c2c9051dc4f80ef1a091ace0085c5b0cf8f28dc7 Mon Sep 17 00:00:00 2001 From: Rob Morgan Date: Tue, 14 May 2019 15:56:36 +0200 Subject: [PATCH 4/9] create a blank kubectl config --- .circleci/config.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8ad93de..a13cf52 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -94,6 +94,11 @@ jobs: sudo apt-get remove -y google-cloud-sdk sudo /opt/google-cloud-sdk/bin/gcloud --quiet components update sudo /opt/google-cloud-sdk/bin/gcloud --quiet components update beta kubectl + - run: + name: configure kubectl + comand: | + mkdir -p ${HOME}/.kube + touch ${HOME}/.kube/config - run: name: run tests command: | From cc3d53b2633088fdc047a00fe73ed8b88f08134c Mon Sep 17 00:00:00 2001 From: Rob Morgan Date: Tue, 14 May 2019 16:06:31 +0200 Subject: [PATCH 5/9] fix typo --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a13cf52..94fce18 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -96,7 +96,7 @@ jobs: sudo /opt/google-cloud-sdk/bin/gcloud --quiet components update beta kubectl - run: name: configure kubectl - comand: | + command: | mkdir -p ${HOME}/.kube touch ${HOME}/.kube/config - run: From bee4091ce8db8e300c4f634fddaff275ca55f76c Mon Sep 17 00:00:00 2001 From: Rob Morgan Date: Tue, 14 May 2019 17:34:52 +0200 Subject: [PATCH 6/9] configure kubectl so it can find the worker nodes --- test/gke_basic_tiller_test.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/test/gke_basic_tiller_test.go b/test/gke_basic_tiller_test.go index ed3d5e7..28a5f49 100644 --- a/test/gke_basic_tiller_test.go +++ b/test/gke_basic_tiller_test.go @@ -14,6 +14,7 @@ import ( "github.com/gruntwork-io/terratest/modules/k8s" "github.com/gruntwork-io/terratest/modules/logger" "github.com/gruntwork-io/terratest/modules/random" + "github.com/gruntwork-io/terratest/modules/shell" "github.com/gruntwork-io/terratest/modules/terraform" test_structure "github.com/gruntwork-io/terratest/modules/test-structure" "github.com/stretchr/testify/require" @@ -26,6 +27,7 @@ func TestGKEBasicTiller(t *testing.T) { // os.Setenv("SKIP_create_test_copy_of_examples", "true") // os.Setenv("SKIP_create_terratest_options", "true") // os.Setenv("SKIP_terraform_apply", "true") + // os.Setenv("SKIP_configure_kubectl", "true") // os.Setenv("SKIP_wait_for_workers", "true") // os.Setenv("SKIP_helm_install", "true") // os.Setenv("SKIP_cleanup", "true") @@ -70,6 +72,25 @@ func TestGKEBasicTiller(t *testing.T) { terraform.InitAndApply(t, gkeClusterTerratestOptions) }) + test_structure.RunTestStage(t, "configure_kubectl", func() { + gkeClusterTerratestOptions := test_structure.LoadTerraformOptions(t, workingDir) + kubectlOptions := test_structure.LoadKubectlOptions(t, workingDir) + project := test_structure.LoadString(t, workingDir, "project") + region := test_structure.LoadString(t, workingDir, "region") + clusterName := gkeClusterTerratestOptions.Vars["cluster_name"].(string) + + // gcloud beta container clusters get-credentials example-cluster --region australia-southeast1 --project dev-sandbox-123456 + cmd := shell.Command{ + Command: "gcloud", + Args: []string{"beta", "container", "clusters", "get-credentials", clusterName, "--region", region, "--project", project}, + Env: map[string]string{ + "KUBECONFIG": kubectlOptions.ConfigPath, + }, + } + + shell.RunCommand(t, cmd) + }) + test_structure.RunTestStage(t, "wait_for_workers", func() { kubectlOptions := test_structure.LoadKubectlOptions(t, workingDir) verifyGkeNodesAreReady(t, kubectlOptions) From 400ae3ad370002efb0527e43b8cf7a2f2aaf79a9 Mon Sep 17 00:00:00 2001 From: Rob Morgan Date: Tue, 14 May 2019 18:32:33 +0200 Subject: [PATCH 7/9] ensure all commands pass auth --- examples/gke-basic-tiller/main.tf | 15 ++++++++++++++- examples/gke-basic-tiller/variables.tf | 7 +++++++ test/gke_basic_tiller_test.go | 21 --------------------- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/examples/gke-basic-tiller/main.tf b/examples/gke-basic-tiller/main.tf index c3cf63d..910cd7e 100644 --- a/examples/gke-basic-tiller/main.tf +++ b/examples/gke-basic-tiller/main.tf @@ -57,8 +57,9 @@ data "google_client_config" "client" {} data "google_client_openid_userinfo" "terraform_user" {} provider "kubernetes" { - load_config_file = false + version = "~> 1.5.2" + load_config_file = false host = "${data.template_file.gke_host_endpoint.rendered}" token = "${data.template_file.access_token.rendered}" cluster_ca_certificate = "${data.template_file.cluster_ca_certificate.rendered}" @@ -207,6 +208,11 @@ module "vpc_network" { resource "null_resource" "configure_kubectl" { provisioner "local-exec" { command = "gcloud beta container clusters get-credentials ${module.gke_cluster.name} --region ${var.region} --project ${var.project}" + + # Use environment variables to allow custom kubectl config paths + environment = { + KUBECONFIG = "${var.kubectl_config_path != "" ? "${var.kubectl_config_path}" : ""}" + } } depends_on = ["google_container_node_pool.node_pool"] @@ -330,6 +336,13 @@ resource "null_resource" "grant_and_configure_helm" { kubergrunt helm configure --helm-home ${pathexpand("~/.helm")} --tiller-namespace ${local.tiller_namespace} --resource-namespace ${local.resource_namespace} --rbac-user ${data.google_client_openid_userinfo.terraform_user.email} ${local.kubectl_auth_config} EOF + + # Use environment variables for Kubernetes credentials to avoid leaking into the logs + environment = { + KUBECTL_SERVER_ENDPOINT = "${data.template_file.gke_host_endpoint.rendered}" + KUBECTL_CA_DATA = "${base64encode(data.template_file.cluster_ca_certificate.rendered)}" + KUBECTL_TOKEN = "${data.template_file.access_token.rendered}" + } } depends_on = ["null_resource.wait_for_tiller"] diff --git a/examples/gke-basic-tiller/variables.tf b/examples/gke-basic-tiller/variables.tf index baf4a12..1cb4b44 100644 --- a/examples/gke-basic-tiller/variables.tf +++ b/examples/gke-basic-tiller/variables.tf @@ -35,6 +35,13 @@ variable "cluster_service_account_description" { default = "Example GKE Cluster Service Account managed by Terraform" } +# Kubectl options + +variable "kubectl_config_path" { + description = "Path to the kubectl config file. Defaults to $HOME/.kube/config" + default = "" +} + # Tiller TLS settings variable "tls_subject" { diff --git a/test/gke_basic_tiller_test.go b/test/gke_basic_tiller_test.go index 28a5f49..ed3d5e7 100644 --- a/test/gke_basic_tiller_test.go +++ b/test/gke_basic_tiller_test.go @@ -14,7 +14,6 @@ import ( "github.com/gruntwork-io/terratest/modules/k8s" "github.com/gruntwork-io/terratest/modules/logger" "github.com/gruntwork-io/terratest/modules/random" - "github.com/gruntwork-io/terratest/modules/shell" "github.com/gruntwork-io/terratest/modules/terraform" test_structure "github.com/gruntwork-io/terratest/modules/test-structure" "github.com/stretchr/testify/require" @@ -27,7 +26,6 @@ func TestGKEBasicTiller(t *testing.T) { // os.Setenv("SKIP_create_test_copy_of_examples", "true") // os.Setenv("SKIP_create_terratest_options", "true") // os.Setenv("SKIP_terraform_apply", "true") - // os.Setenv("SKIP_configure_kubectl", "true") // os.Setenv("SKIP_wait_for_workers", "true") // os.Setenv("SKIP_helm_install", "true") // os.Setenv("SKIP_cleanup", "true") @@ -72,25 +70,6 @@ func TestGKEBasicTiller(t *testing.T) { terraform.InitAndApply(t, gkeClusterTerratestOptions) }) - test_structure.RunTestStage(t, "configure_kubectl", func() { - gkeClusterTerratestOptions := test_structure.LoadTerraformOptions(t, workingDir) - kubectlOptions := test_structure.LoadKubectlOptions(t, workingDir) - project := test_structure.LoadString(t, workingDir, "project") - region := test_structure.LoadString(t, workingDir, "region") - clusterName := gkeClusterTerratestOptions.Vars["cluster_name"].(string) - - // gcloud beta container clusters get-credentials example-cluster --region australia-southeast1 --project dev-sandbox-123456 - cmd := shell.Command{ - Command: "gcloud", - Args: []string{"beta", "container", "clusters", "get-credentials", clusterName, "--region", region, "--project", project}, - Env: map[string]string{ - "KUBECONFIG": kubectlOptions.ConfigPath, - }, - } - - shell.RunCommand(t, cmd) - }) - test_structure.RunTestStage(t, "wait_for_workers", func() { kubectlOptions := test_structure.LoadKubectlOptions(t, workingDir) verifyGkeNodesAreReady(t, kubectlOptions) From 7c52798350e21f9e084f7fcaebba424592268dd6 Mon Sep 17 00:00:00 2001 From: Rob Morgan Date: Tue, 14 May 2019 21:35:29 +0200 Subject: [PATCH 8/9] be sure to pass in kubectl config --- test/gke_basic_tiller_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/test/gke_basic_tiller_test.go b/test/gke_basic_tiller_test.go index ed3d5e7..258955f 100644 --- a/test/gke_basic_tiller_test.go +++ b/test/gke_basic_tiller_test.go @@ -103,6 +103,7 @@ func TestGKEBasicTiller(t *testing.T) { "HELM_TLS_VERIFY": "true", "HELM_TLS_ENABLE": "true", }, + KubectlOptions: kubectlOptions, } // Deploy the chart using `helm install`. Note that we use the version without `E`, since we want to assert the From 4eb2d64820244b807c6c8212eb5036342c3bc46e Mon Sep 17 00:00:00 2001 From: Rob Morgan Date: Tue, 14 May 2019 21:41:52 +0200 Subject: [PATCH 9/9] ensure the root example has the updated code --- main.tf | 15 ++++++++++++++- variables.tf | 7 +++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 69ea747..da1e6ef 100644 --- a/main.tf +++ b/main.tf @@ -56,8 +56,9 @@ data "google_client_config" "client" {} data "google_client_openid_userinfo" "terraform_user" {} provider "kubernetes" { - load_config_file = false + version = "~> 1.5.2" + load_config_file = false host = "${data.template_file.gke_host_endpoint.rendered}" token = "${data.template_file.access_token.rendered}" cluster_ca_certificate = "${data.template_file.cluster_ca_certificate.rendered}" @@ -224,6 +225,11 @@ module "vpc_network" { resource "null_resource" "configure_kubectl" { provisioner "local-exec" { command = "gcloud beta container clusters get-credentials ${module.gke_cluster.name} --region ${var.region} --project ${var.project}" + + # Use environment variables to allow custom kubectl config paths + environment = { + KUBECONFIG = "${var.kubectl_config_path != "" ? "${var.kubectl_config_path}" : ""}" + } } depends_on = ["google_container_node_pool.node_pool"] @@ -347,6 +353,13 @@ resource "null_resource" "grant_and_configure_helm" { kubergrunt helm configure --helm-home ${pathexpand("~/.helm")} --tiller-namespace ${local.tiller_namespace} --resource-namespace ${local.resource_namespace} --rbac-user ${data.google_client_openid_userinfo.terraform_user.email} ${local.kubectl_auth_config} EOF + + # Use environment variables for Kubernetes credentials to avoid leaking into the logs + environment = { + KUBECTL_SERVER_ENDPOINT = "${data.template_file.gke_host_endpoint.rendered}" + KUBECTL_CA_DATA = "${base64encode(data.template_file.cluster_ca_certificate.rendered)}" + KUBECTL_TOKEN = "${data.template_file.access_token.rendered}" + } } depends_on = ["null_resource.wait_for_tiller"] diff --git a/variables.tf b/variables.tf index 9ffcb30..5b6b79e 100644 --- a/variables.tf +++ b/variables.tf @@ -35,6 +35,13 @@ variable "cluster_service_account_description" { default = "Example GKE Cluster Service Account managed by Terraform" } +# Kubectl options + +variable "kubectl_config_path" { + description = "Path to the kubectl config file. Defaults to $HOME/.kube/config" + default = "" +} + # Tiller TLS settings variable "tls_subject" {