From 41740677a70a049fe85a26260439cede2d512d84 Mon Sep 17 00:00:00 2001 From: Rob Morgan Date: Thu, 21 Feb 2019 17:42:19 +0100 Subject: [PATCH 01/15] add basic helm on gke example --- examples/gke-basic-tiller/dependencies.tf | 20 ++ examples/gke-basic-tiller/main.tf | 219 ++++++++++++++++++++++ examples/gke-basic-tiller/outputs.tf | 22 +++ examples/gke-basic-tiller/variables.tf | 81 ++++++++ test/gke_basic_tiller_test.go | 66 +++++++ test/gke_cluster_test.go | 11 -- test/terratest_options.go | 11 ++ test/test_helpers.go | 9 + 8 files changed, 428 insertions(+), 11 deletions(-) create mode 100644 examples/gke-basic-tiller/dependencies.tf create mode 100644 examples/gke-basic-tiller/main.tf create mode 100644 examples/gke-basic-tiller/outputs.tf create mode 100644 examples/gke-basic-tiller/variables.tf create mode 100644 test/gke_basic_tiller_test.go diff --git a/examples/gke-basic-tiller/dependencies.tf b/examples/gke-basic-tiller/dependencies.tf new file mode 100644 index 0000000..689dc72 --- /dev/null +++ b/examples/gke-basic-tiller/dependencies.tf @@ -0,0 +1,20 @@ +# --------------------------------------------------------------------------------------------------------------------- +# INTERPOLATE AND CONSTRUCT COMMAND ARGUMENTS +# --------------------------------------------------------------------------------------------------------------------- + +locals { + tls_config = "--tls-private-key-algorithm ${var.private_key_algorithm} ${local.tls_algorithm_config} --tls-common-name ${lookup(var.tls_subject, "common_name")} --tls-org ${lookup(var.tls_subject, "org")} ${local.tls_org_unit} ${local.tls_city} ${local.tls_state} ${local.tls_country}" + tls_algorithm_config = "${var.private_key_algorithm == "ECDSA" ? "--tls-private-key-ecdsa-curve ${var.private_key_ecdsa_curve}" : "--tls-private-key-rsa-bits ${var.private_key_rsa_bits}"}" + tls_org_unit = "${lookup(var.tls_subject, "org_unit", "") != "" ? "--tls-org-unit ${lookup(var.tls_subject, "org_unit", "")}" : ""}" + tls_city = "${lookup(var.tls_subject, "city", "") != "" ? "--tls-city ${lookup(var.tls_subject, "city", "")}" : ""}" + tls_state = "${lookup(var.tls_subject, "state", "") != "" ? "--tls-state ${lookup(var.tls_subject, "state", "")}" : ""}" + tls_country = "${lookup(var.tls_subject, "country", "") != "" ? "--tls-country ${lookup(var.tls_subject, "country", "")}" : ""}" + + client_tls_config = "--client-tls-common-name ${lookup(var.client_tls_subject, "common_name")} --client-tls-org ${lookup(var.client_tls_subject, "org")} ${local.client_tls_org_unit} ${local.client_tls_city} ${local.client_tls_state} ${local.client_tls_country}" + client_tls_org_unit = "${lookup(var.client_tls_subject, "org_unit", "") != "" ? "--client-tls-org-unit ${lookup(var.client_tls_subject, "org_unit", "")}" : ""}" + client_tls_city = "${lookup(var.client_tls_subject, "city", "") != "" ? "--client-tls-city ${lookup(var.client_tls_subject, "city", "")}" : ""}" + client_tls_state = "${lookup(var.client_tls_subject, "state", "") != "" ? "--client-tls-state ${lookup(var.client_tls_subject, "state", "")}" : ""}" + client_tls_country = "${lookup(var.client_tls_subject, "country", "") != "" ? "--client-tls-country ${lookup(var.client_tls_subject, "country", "")}" : ""}" + + undeploy_args = "${var.force_undeploy ? "--force" : ""} ${var.undeploy_releases ? "--undeploy-releases" : ""}" +} diff --git a/examples/gke-basic-tiller/main.tf b/examples/gke-basic-tiller/main.tf new file mode 100644 index 0000000..10d3068 --- /dev/null +++ b/examples/gke-basic-tiller/main.tf @@ -0,0 +1,219 @@ +# --------------------------------------------------------------------------------------------------------------------- +# DEPLOY A GKE REGIONAL PUBLIC CLUSTER IN GOOGLE CLOUD +# This is an example of how to use the gke-cluster module to deploy a regional public Kubernetes cluster in GCP with a +# Load Balancer in front of it. +# --------------------------------------------------------------------------------------------------------------------- + +# Use Terraform 0.10.x so that we can take advantage of Terraform GCP functionality as a separate provider via +# https://github.com/terraform-providers/terraform-provider-google +terraform { + required_version = ">= 0.10.3" +} + +provider "google-beta" { + project = "${var.project}" + region = "${var.region}" +} + +# We use this data provider to expose an access token for communicating with the GKE cluster. +data "google_client_config" "client" {} + +provider "kubernetes" { + load_config_file = false + + host = "${data.template_file.gke_host_endpoint.rendered}" + token = "${data.template_file.access_token.rendered}" + cluster_ca_certificate = "${data.template_file.cluster_ca_certificate.rendered}" +} + +provider "helm" { + # We don't install Tiller automatically, but instead use Kubergrunt as it sets up the TLS certificates much easier. + install_tiller = false + + # Enable TLS so Helm can communicate with Tiller securely. + enable_tls = true + + # We can remove the following parameters after Yori's PR is released: + # https://github.com/terraform-providers/terraform-provider-helm/pull/210 + client_key = "${pathexpand("~/.helm/key.pem")}" + + client_certificate = "${pathexpand("~/.helm/cert.pem")}" + ca_certificate = "${pathexpand("~/.helm/ca.pem")}" + + kubernetes { + host = "${data.template_file.gke_host_endpoint.rendered}" + token = "${data.template_file.access_token.rendered}" + cluster_ca_certificate = "${data.template_file.cluster_ca_certificate.rendered}" + } +} + +# --------------------------------------------------------------------------------------------------------------------- +# DEPLOY A GKE REGIONAL PUBLIC CLUSTER IN GOOGLE CLOUD +# --------------------------------------------------------------------------------------------------------------------- + +module "gke_cluster" { + # When using these modules in your own templates, you will need to use a Git URL with a ref attribute that pins you + # to a specific version of the modules, such as the following example: + # source = "git::git@github.com:gruntwork-io/gke-cluster.git//modules/gke-cluster?ref=v0.0.1" + source = "../../modules/gke-cluster" + + name = "${var.cluster_name}" + + project = "${var.project}" + region = "${var.region}" + network = "${google_compute_network.main.name}" + subnetwork = "${google_compute_subnetwork.main.name}" +} + +# Deploy a Node Pool + +resource "google_container_node_pool" "node_pool" { + provider = "google-beta" + + name = "main-pool" + project = "${var.project}" + region = "${var.region}" + cluster = "${module.gke_cluster.name}" + + initial_node_count = "1" + + autoscaling { + min_node_count = "1" + max_node_count = "5" + } + + management { + auto_repair = "true" + auto_upgrade = "true" + } + + node_config { + image_type = "COS" + machine_type = "n1-standard-1" + + labels = { + all-pools-example = "true" + } + + tags = ["main-pool-example"] + disk_size_gb = "30" + disk_type = "pd-standard" + preemptible = false + + oauth_scopes = [ + "https://www.googleapis.com/auth/cloud-platform", + ] + } + + lifecycle { + ignore_changes = ["initial_node_count"] + } + + timeouts { + create = "30m" + update = "30m" + delete = "30m" + } +} + +# TODO(rileykarson): Add proper VPC network config once we've made a VPC module +resource "random_string" "suffix" { + length = 4 + special = false + upper = false +} + +resource "google_compute_network" "main" { + name = "${var.cluster_name}-network-${random_string.suffix.result}" + auto_create_subnetworks = "false" +} + +resource "google_compute_subnetwork" "main" { + name = "${var.cluster_name}-subnetwork-${random_string.suffix.result}" + ip_cidr_range = "10.0.0.0/17" + region = "${var.region}" + network = "${google_compute_network.main.self_link}" +} + +# --------------------------------------------------------------------------------------------------------------------- +# CONFIGURE KUBECTL AND RBAC ROLE PERMISSIONS +# --------------------------------------------------------------------------------------------------------------------- + +# configure kubectl with the credentials of the GKE cluster +resource "null_resource" "configure_kubectl" { + provisioner "local-exec" { + command = "gcloud beta container clusters get-credentials ${module.gke_cluster.name} --region ${var.region} --project ${var.project}" + } + + depends_on = ["google_container_node_pool.node_pool"] +} + +resource "kubernetes_cluster_role_binding" "user" { + metadata { + name = "admin-user" + } + + role_ref { + kind = "ClusterRole" + name = "cluster-admin" + api_group = "rbac.authorization.k8s.io" + } + + subject { + kind = "User" + name = "${var.iam_user}" + api_group = "rbac.authorization.k8s.io" + } + + subject { + # this is a workaround for https://github.com/terraform-providers/terraform-provider-kubernetes/issues/204. + # we have to set an empty api_group or the k8s call will fail. + api_group = "" + + kind = "ServiceAccount" + name = "default" + namespace = "kube-system" + } + + subject { + kind = "Group" + name = "system:masters" + api_group = "rbac.authorization.k8s.io" + } +} + +# --------------------------------------------------------------------------------------------------------------------- +# DEPLOY TILLER TO THE GKE CLUSTER USING KUBERGRUNT +# --------------------------------------------------------------------------------------------------------------------- + +# use an old version of Tiller because the provider expects this +resource "null_resource" "tiller" { + provisioner "local-exec" { + command = "kubergrunt helm deploy --service-account default --resource-namespace default --tiller-namespace kube-system ${local.tls_config} ${local.client_tls_config} --helm-home ${pathexpand("~/.helm")} --tiller-version v2.11.0 --rbac-user ${var.iam_user}" + } + + provisioner "local-exec" { + command = "kubergrunt helm undeploy --helm-home ${pathexpand("~/.helm")} --tiller-namespace kube-system ${local.undeploy_args}" + when = "destroy" + } + + depends_on = ["null_resource.configure_kubectl", "kubernetes_cluster_role_binding.user"] +} + +# --------------------------------------------------------------------------------------------------------------------- +# WORKAROUNDS +# --------------------------------------------------------------------------------------------------------------------- + +# This is a workaround for the Kubernetes and Helm providers as Terraform doesn't currently support passing in module +# outputs to providers directly. +data "template_file" "gke_host_endpoint" { + template = "${module.gke_cluster.endpoint}" +} + +data "template_file" "access_token" { + template = "${data.google_client_config.client.access_token}" +} + +data "template_file" "cluster_ca_certificate" { + template = "${module.gke_cluster.cluster_ca_certificate}" +} diff --git a/examples/gke-basic-tiller/outputs.tf b/examples/gke-basic-tiller/outputs.tf new file mode 100644 index 0000000..51f473b --- /dev/null +++ b/examples/gke-basic-tiller/outputs.tf @@ -0,0 +1,22 @@ +output "cluster_endpoint" { + description = "The IP address of the cluster master." + sensitive = true + value = "${module.gke_cluster.endpoint}" +} + +output "client_certificate" { + description = "Public certificate used by clients to authenticate to the cluster endpoint." + value = "${module.gke_cluster.client_certificate}" +} + +output "client_key" { + description = "Private key used by clients to authenticate to the cluster endpoint." + sensitive = true + value = "${module.gke_cluster.client_key}" +} + +output "cluster_ca_certificate" { + description = "The public certificate that is the root of trust for the cluster." + sensitive = true + value = "${module.gke_cluster.cluster_ca_certificate}" +} diff --git a/examples/gke-basic-tiller/variables.tf b/examples/gke-basic-tiller/variables.tf new file mode 100644 index 0000000..8708af2 --- /dev/null +++ b/examples/gke-basic-tiller/variables.tf @@ -0,0 +1,81 @@ +# --------------------------------------------------------------------------------------------------------------------- +# REQUIRED PARAMETERS +# These parameters must be supplied when consuming this module. +# --------------------------------------------------------------------------------------------------------------------- + +variable "project" { + description = "The name of the GCP Project where all resources will be launched." +} + +variable "region" { + description = "The Region in which all GCP resources will be launched." +} + +variable "iam_user" { + description = "The name of the IAM user (email address) that will be granted the ability to create Kubernetes roles." +} + +variable "tls_subject" { + description = "The issuer information that contains the identifying information for the Tiller server. Used to generate the TLS certificate keypairs." + type = "map" + + # Expects the following keys + # - common_name + # - org + # - org_unit + # - city + # - state + # - country +} + +variable "client_tls_subject" { + description = "The issuer information that contains the identifying information for the helm client of the operator. Used to generate the TLS certificate keypairs." + type = "map" + + # Expects the following keys + # - common_name + # - org + # - org_unit + # - city + # - state + # - country +} + +# --------------------------------------------------------------------------------------------------------------------- +# OPTIONAL PARAMETERS +# These parameters have reasonable defaults. +# --------------------------------------------------------------------------------------------------------------------- + +variable "cluster_name" { + description = "The name of the Kubernetes cluster." + default = "example-cluster" +} + +# TLS algorithm configuration + +variable "private_key_algorithm" { + description = "The name of the algorithm to use for private keys. Must be one of: RSA or ECDSA." + default = "ECDSA" +} + +variable "private_key_ecdsa_curve" { + description = "The name of the elliptic curve to use. Should only be used if var.private_key_algorithm is ECDSA. Must be one of P224, P256, P384 or P521." + default = "P256" +} + +variable "private_key_rsa_bits" { + description = "The size of the generated RSA key in bits. Should only be used if var.private_key_algorithm is RSA." + default = "2048" +} + +# Undeploy options + +variable "force_undeploy" { + description = "If true, will remove the Tiller server resources even if there are releases deployed." + default = false +} + +variable "undeploy_releases" { + description = "If true, will delete deployed releases from the Tiller instance before undeploying Tiller." + default = false +} diff --git a/test/gke_basic_tiller_test.go b/test/gke_basic_tiller_test.go new file mode 100644 index 0000000..d00ef4b --- /dev/null +++ b/test/gke_basic_tiller_test.go @@ -0,0 +1,66 @@ +package test + +import ( + "path/filepath" + "testing" + + "github.com/gruntwork-io/terratest/modules/gcp" + "github.com/gruntwork-io/terratest/modules/logger" + "github.com/gruntwork-io/terratest/modules/random" + "github.com/gruntwork-io/terratest/modules/terraform" + "github.com/gruntwork-io/terratest/modules/test-structure" +) + +func TestGKEBasicTiller(t *testing.T) { + t.Parallel() + + // Uncomment any of the following to skip that section during the test + // os.Setenv("SKIP_create_test_copy_of_examples", "true") + // os.Setenv("SKIP_create_terratest_options", "true") + // os.Setenv("SKIP_terraform_apply", "true") + // os.Setenv("SKIP_configure_kubectl", "true") + // os.Setenv("SKIP_wait_for_workers", "true") + // os.Setenv("SKIP_cleanup", "true") + + // - [x] Create GKE cluster + // - [x] Configure kubectl + // - [x] Securely install Helm using terraform-kubernetes-helm + // - [ ] Install an example chart (nginx?) + // - [ ] test port 80 open/running + + // Create a directory path that won't conflict + workingDir := filepath.Join(".", "stages", t.Name()) + + test_structure.RunTestStage(t, "create_test_copy_of_examples", func() { + testFolder := test_structure.CopyTerraformFolderToTemp(t, "..", "examples") + logger.Logf(t, "path to test folder %s\n", testFolder) + terraformModulePath := filepath.Join(testFolder, "gke-basic-tiller") + test_structure.SaveString(t, workingDir, "gkeBasicTillerTerraformModulePath", terraformModulePath) + }) + + test_structure.RunTestStage(t, "create_terratest_options", func() { + gkeBasicTillerTerraformModulePath := test_structure.LoadString(t, workingDir, "gkeBasicTillerTerraformModulePath") + uniqueID := random.UniqueId() + project := gcp.GetGoogleProjectIDFromEnvVar(t) + region := gcp.GetRandomRegion(t, project, nil, nil) + gkeClusterTerratestOptions := createGKEClusterTerraformOptions(t, uniqueID, project, region, gkeBasicTillerTerraformModulePath) + test_structure.SaveString(t, workingDir, "uniqueID", uniqueID) + test_structure.SaveString(t, workingDir, "project", project) + test_structure.SaveString(t, workingDir, "region", region) + test_structure.SaveTerraformOptions(t, workingDir, gkeClusterTerratestOptions) + }) + + defer test_structure.RunTestStage(t, "cleanup", func() { + gkeClusterTerratestOptions := test_structure.LoadTerraformOptions(t, workingDir) + terraform.Destroy(t, gkeClusterTerratestOptions) + }) + + test_structure.RunTestStage(t, "terraform_apply", func() { + gkeClusterTerratestOptions := test_structure.LoadTerraformOptions(t, workingDir) + terraform.InitAndApply(t, gkeClusterTerratestOptions) + }) + + test_structure.RunTestStage(t, "wait_for_workers", func() { + verifyGkeNodesAreReady(t) + }) +} diff --git a/test/gke_cluster_test.go b/test/gke_cluster_test.go index 5184083..8b1cf4a 100644 --- a/test/gke_cluster_test.go +++ b/test/gke_cluster_test.go @@ -3,16 +3,13 @@ package test import ( "path/filepath" "testing" - "time" "github.com/gruntwork-io/terratest/modules/gcp" - "github.com/gruntwork-io/terratest/modules/k8s" "github.com/gruntwork-io/terratest/modules/logger" "github.com/gruntwork-io/terratest/modules/random" "github.com/gruntwork-io/terratest/modules/shell" "github.com/gruntwork-io/terratest/modules/terraform" "github.com/gruntwork-io/terratest/modules/test-structure" - "github.com/stretchr/testify/assert" ) func TestGKECluster(t *testing.T) { @@ -77,11 +74,3 @@ func TestGKECluster(t *testing.T) { verifyGkeNodesAreReady(t) }) } - -// Verify that all the nodes in the cluster reach the Ready state. -func verifyGkeNodesAreReady(t *testing.T) { - kubeWaitUntilNumNodes(t, 3, 30, 10*time.Second) - k8s.WaitUntilAllNodesReady(t, 30, 10*time.Second) - readyNodes := k8s.GetReadyNodes(t) - assert.Equal(t, len(readyNodes), 3) -} diff --git a/test/terratest_options.go b/test/terratest_options.go index 4db8a23..440794f 100644 --- a/test/terratest_options.go +++ b/test/terratest_options.go @@ -21,6 +21,17 @@ func createGKEClusterTerraformOptions( "region": region, "project": project, "cluster_name": gkeClusterName, + "iam_user": "rob@gruntwork.io", + "tls_subject": map[string]string{ + "common_name": "tiller", + "org": "Gruntwork", + }, + "client_tls_subject": map[string]string{ + "common_name": "rob@gruntwork.io", + "org": "Gruntwork", + }, + "force_undeploy": true, + "undeploy_release": true, } terratestOptions := terraform.Options{ diff --git a/test/test_helpers.go b/test/test_helpers.go index 127eeb9..f2d51ed 100644 --- a/test/test_helpers.go +++ b/test/test_helpers.go @@ -9,6 +9,7 @@ import ( "github.com/gruntwork-io/terratest/modules/k8s" "github.com/gruntwork-io/terratest/modules/logger" "github.com/gruntwork-io/terratest/modules/retry" + "github.com/stretchr/testify/assert" ) // kubeWaitUntilNumNodes continuously polls the Kubernetes cluster until there are the expected number of nodes @@ -37,3 +38,11 @@ func kubeWaitUntilNumNodes(t *testing.T, numNodes int, retries int, sleepBetween } logger.Logf(t, message) } + +// Verify that all the nodes in the cluster reach the Ready state. +func verifyGkeNodesAreReady(t *testing.T) { + kubeWaitUntilNumNodes(t, 3, 30, 10*time.Second) + k8s.WaitUntilAllNodesReady(t, 30, 10*time.Second) + readyNodes := k8s.GetReadyNodes(t) + assert.Equal(t, len(readyNodes), 3) +} From a49908fe144d4424bd5796910790b8a4d56cdab0 Mon Sep 17 00:00:00 2001 From: Rob Morgan Date: Thu, 21 Feb 2019 18:10:05 +0100 Subject: [PATCH 02/15] improve docs --- examples/gke-basic-tiller/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/gke-basic-tiller/main.tf b/examples/gke-basic-tiller/main.tf index 10d3068..db84d98 100644 --- a/examples/gke-basic-tiller/main.tf +++ b/examples/gke-basic-tiller/main.tf @@ -186,7 +186,7 @@ resource "kubernetes_cluster_role_binding" "user" { # DEPLOY TILLER TO THE GKE CLUSTER USING KUBERGRUNT # --------------------------------------------------------------------------------------------------------------------- -# use an old version of Tiller because the provider expects this +# We install an older version of Tiller as the provider expects this resource "null_resource" "tiller" { provisioner "local-exec" { command = "kubergrunt helm deploy --service-account default --resource-namespace default --tiller-namespace kube-system ${local.tls_config} ${local.client_tls_config} --helm-home ${pathexpand("~/.helm")} --tiller-version v2.11.0 --rbac-user ${var.iam_user}" From 965a2c702170c4e5576c1938451bd96ff6705a5a Mon Sep 17 00:00:00 2001 From: Rob Morgan Date: Thu, 21 Feb 2019 18:10:44 +0100 Subject: [PATCH 03/15] ensure kubergrunt and helm are installed --- .circleci/config.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9d883d6..378cfb9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,6 +3,8 @@ defaults: &defaults environment: GRUNTWORK_INSTALLER_VERSION: v0.0.21 TERRATEST_LOG_PARSER_VERSION: v0.13.13 + KUBERGRUNT_VERSION: v0.3.1 + HELM_VERSION: v2.11.0 MODULE_CI_VERSION: v0.13.3 TERRAFORM_VERSION: 0.11.8 TERRAGRUNT_VERSION: NONE @@ -26,6 +28,15 @@ install_gruntwork_utils: &install_gruntwork_utils --go-version ${GOLANG_VERSION} \ --go-src-path test +install_helm: &install_helm + name: install helm + command: | + # install helm + curl -Lo helm.tar.gz https://storage.googleapis.com/kubernetes-helm/helm-${HELM_VERSION}-linux-amd64.tar.gz + tar -xvf helm.tar.gz + chmod +x linux-amd64/helm + sudo mv linux-amd64/helm /usr/local/bin/ + version: 2 jobs: build: @@ -40,6 +51,16 @@ jobs: - run: <<: *install_gruntwork_utils + # Install helm + - run: + <<: *install_helm + + # Install kubergrunt + - run: + name: Install kubergrunt + command: gruntwork-install --binary-name "kubergrunt" --repo "https://github.com/gruntwork-io/kubergrunt" --tag "${KUBERGRUNT_VERSION}" + + - save_cache: key: dep-v1-{{ checksum "test/Gopkg.lock" }} paths: From 05e88c9c992ca9d5b88033d65dcfbb512a41b048 Mon Sep 17 00:00:00 2001 From: Rob Morgan Date: Thu, 21 Feb 2019 18:25:33 +0100 Subject: [PATCH 04/15] move utils to test section --- .circleci/config.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 378cfb9..1e68eb1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -51,16 +51,6 @@ jobs: - run: <<: *install_gruntwork_utils - # Install helm - - run: - <<: *install_helm - - # Install kubergrunt - - run: - name: Install kubergrunt - command: gruntwork-install --binary-name "kubergrunt" --repo "https://github.com/gruntwork-io/kubergrunt" --tag "${KUBERGRUNT_VERSION}" - - - save_cache: key: dep-v1-{{ checksum "test/Gopkg.lock" }} paths: @@ -87,7 +77,17 @@ jobs: - checkout - run: echo 'export PATH=$HOME/terraform:$HOME/packer:$PATH' >> $BASH_ENV - run: - <<: *install_gruntwork_utils + <<: *install_gruntwork_utils + + # Install helm + - run: + <<: *install_helm + + # Install kubergrunt + - run: + name: Install kubergrunt + command: gruntwork-install --binary-name "kubergrunt" --repo "https://github.com/gruntwork-io/kubergrunt" --tag "${KUBERGRUNT_VERSION}" + - run: name: update gcloud command: | From 22170b0559e3928a802445baa3615da0efed0be0 Mon Sep 17 00:00:00 2001 From: Rob Morgan Date: Fri, 22 Feb 2019 10:32:12 +0100 Subject: [PATCH 05/15] bump kubergrunt to 0.3.2 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1e68eb1..c36afb7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,7 +3,7 @@ defaults: &defaults environment: GRUNTWORK_INSTALLER_VERSION: v0.0.21 TERRATEST_LOG_PARSER_VERSION: v0.13.13 - KUBERGRUNT_VERSION: v0.3.1 + KUBERGRUNT_VERSION: v0.3.2 HELM_VERSION: v2.11.0 MODULE_CI_VERSION: v0.13.3 TERRAFORM_VERSION: 0.11.8 From 04bf1222cf9836c5f407bcd5c4d7e6aab6cd7fa9 Mon Sep 17 00:00:00 2001 From: Rob Morgan Date: Fri, 22 Feb 2019 11:25:04 +0100 Subject: [PATCH 06/15] change the tests to get the iam user from the env --- test/gke_basic_tiller_test.go | 4 +++- test/gke_cluster_test.go | 4 +++- test/terratest_options.go | 5 +++-- test/test_helpers.go | 12 ++++++++++++ 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/test/gke_basic_tiller_test.go b/test/gke_basic_tiller_test.go index d00ef4b..0fce54e 100644 --- a/test/gke_basic_tiller_test.go +++ b/test/gke_basic_tiller_test.go @@ -43,10 +43,12 @@ func TestGKEBasicTiller(t *testing.T) { uniqueID := random.UniqueId() project := gcp.GetGoogleProjectIDFromEnvVar(t) region := gcp.GetRandomRegion(t, project, nil, nil) - gkeClusterTerratestOptions := createGKEClusterTerraformOptions(t, uniqueID, project, region, gkeBasicTillerTerraformModulePath) + iamUser := getIAMUserFromEnv() + gkeClusterTerratestOptions := createGKEClusterTerraformOptions(t, uniqueID, project, region, iamUser, gkeBasicTillerTerraformModulePath) test_structure.SaveString(t, workingDir, "uniqueID", uniqueID) test_structure.SaveString(t, workingDir, "project", project) test_structure.SaveString(t, workingDir, "region", region) + test_structure.SaveString(t, workingDir, "iamUser", iamUser) test_structure.SaveTerraformOptions(t, workingDir, gkeClusterTerratestOptions) }) diff --git a/test/gke_cluster_test.go b/test/gke_cluster_test.go index 8b1cf4a..7d8407d 100644 --- a/test/gke_cluster_test.go +++ b/test/gke_cluster_test.go @@ -38,10 +38,12 @@ func TestGKECluster(t *testing.T) { uniqueID := random.UniqueId() project := gcp.GetGoogleProjectIDFromEnvVar(t) region := gcp.GetRandomRegion(t, project, nil, nil) - gkeClusterTerratestOptions := createGKEClusterTerraformOptions(t, uniqueID, project, region, gkeClusterTerraformModulePath) + iamUser := getIAMUserFromEnv() + gkeClusterTerratestOptions := createGKEClusterTerraformOptions(t, uniqueID, project, region, iamUser, gkeClusterTerraformModulePath) test_structure.SaveString(t, workingDir, "uniqueID", uniqueID) test_structure.SaveString(t, workingDir, "project", project) test_structure.SaveString(t, workingDir, "region", region) + test_structure.SaveString(t, workingDir, "iamUser", iamUser) test_structure.SaveTerraformOptions(t, workingDir, gkeClusterTerratestOptions) }) diff --git a/test/terratest_options.go b/test/terratest_options.go index 440794f..b175c85 100644 --- a/test/terratest_options.go +++ b/test/terratest_options.go @@ -13,6 +13,7 @@ func createGKEClusterTerraformOptions( uniqueID, project string, region string, + iamUser string, templatePath string, ) *terraform.Options { gkeClusterName := strings.ToLower(fmt.Sprintf("gke-cluster-%s", uniqueID)) @@ -21,13 +22,13 @@ func createGKEClusterTerraformOptions( "region": region, "project": project, "cluster_name": gkeClusterName, - "iam_user": "rob@gruntwork.io", + "iam_user": iamUser, "tls_subject": map[string]string{ "common_name": "tiller", "org": "Gruntwork", }, "client_tls_subject": map[string]string{ - "common_name": "rob@gruntwork.io", + "common_name": iamUser, "org": "Gruntwork", }, "force_undeploy": true, diff --git a/test/test_helpers.go b/test/test_helpers.go index f2d51ed..2540e98 100644 --- a/test/test_helpers.go +++ b/test/test_helpers.go @@ -3,6 +3,7 @@ package test import ( "errors" "fmt" + "os" "testing" "time" @@ -12,6 +13,17 @@ import ( "github.com/stretchr/testify/assert" ) +// At the moment we need to inject the current IAM user from the environment. +// If this PR is merged we can use a solution directly within Terraform code: +// https://github.com/GoogleCloudPlatform/magic-modules/pull/1422. +func getIAMUserFromEnv() string { + if value := os.Getenv("GOOGLE_IAM_USER"); value != "" { + return value + } + + return "" +} + // kubeWaitUntilNumNodes continuously polls the Kubernetes cluster until there are the expected number of nodes // registered (regardless of readiness). func kubeWaitUntilNumNodes(t *testing.T, numNodes int, retries int, sleepBetweenRetries time.Duration) { From c61025053968d7ae7a56acfbf1e5a61cb60883b4 Mon Sep 17 00:00:00 2001 From: Rob Morgan Date: Fri, 22 Feb 2019 12:34:23 +0100 Subject: [PATCH 07/15] stop running tests in parallel --- test/gke_basic_tiller_test.go | 2 -- test/gke_cluster_test.go | 2 -- 2 files changed, 4 deletions(-) diff --git a/test/gke_basic_tiller_test.go b/test/gke_basic_tiller_test.go index 0fce54e..7fbf7c9 100644 --- a/test/gke_basic_tiller_test.go +++ b/test/gke_basic_tiller_test.go @@ -12,8 +12,6 @@ import ( ) func TestGKEBasicTiller(t *testing.T) { - t.Parallel() - // Uncomment any of the following to skip that section during the test // os.Setenv("SKIP_create_test_copy_of_examples", "true") // os.Setenv("SKIP_create_terratest_options", "true") diff --git a/test/gke_cluster_test.go b/test/gke_cluster_test.go index 7d8407d..ae5f3b6 100644 --- a/test/gke_cluster_test.go +++ b/test/gke_cluster_test.go @@ -13,8 +13,6 @@ import ( ) func TestGKECluster(t *testing.T) { - t.Parallel() - // Uncomment any of the following to skip that section during the test // os.Setenv("SKIP_create_test_copy_of_examples", "true") // os.Setenv("SKIP_create_terratest_options", "true") From 3de1e4ba3aca91f9f847715d57b55805690f7875 Mon Sep 17 00:00:00 2001 From: Rob Morgan Date: Fri, 22 Feb 2019 13:31:43 +0100 Subject: [PATCH 08/15] rename install helm to install helm client. improve docs --- .circleci/config.yml | 8 ++++---- examples/gke-basic-tiller/main.tf | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c36afb7..8bb9e33 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -28,10 +28,10 @@ install_gruntwork_utils: &install_gruntwork_utils --go-version ${GOLANG_VERSION} \ --go-src-path test -install_helm: &install_helm - name: install helm +install_helm_client: &install_helm_client + name: install helm client command: | - # install helm + # install helm client curl -Lo helm.tar.gz https://storage.googleapis.com/kubernetes-helm/helm-${HELM_VERSION}-linux-amd64.tar.gz tar -xvf helm.tar.gz chmod +x linux-amd64/helm @@ -81,7 +81,7 @@ jobs: # Install helm - run: - <<: *install_helm + <<: *install_helm_client # Install kubergrunt - run: diff --git a/examples/gke-basic-tiller/main.tf b/examples/gke-basic-tiller/main.tf index db84d98..ca2c862 100644 --- a/examples/gke-basic-tiller/main.tf +++ b/examples/gke-basic-tiller/main.tf @@ -186,7 +186,7 @@ resource "kubernetes_cluster_role_binding" "user" { # DEPLOY TILLER TO THE GKE CLUSTER USING KUBERGRUNT # --------------------------------------------------------------------------------------------------------------------- -# We install an older version of Tiller as the provider expects this +# We install an older version of Tiller as the provider expects this. resource "null_resource" "tiller" { provisioner "local-exec" { command = "kubergrunt helm deploy --service-account default --resource-namespace default --tiller-namespace kube-system ${local.tls_config} ${local.client_tls_config} --helm-home ${pathexpand("~/.helm")} --tiller-version v2.11.0 --rbac-user ${var.iam_user}" From 1ec840a0ee057cb6dff4b18f784335d205912f58 Mon Sep 17 00:00:00 2001 From: Rob Morgan Date: Tue, 26 Feb 2019 11:22:19 +0100 Subject: [PATCH 09/15] bump terratest ver --- test/Gopkg.lock | 258 ++++++++++++++++++++++++++++++------------------ test/Gopkg.toml | 2 +- 2 files changed, 164 insertions(+), 96 deletions(-) diff --git a/test/Gopkg.lock b/test/Gopkg.lock index cd04ce2..8dbd30e 100644 --- a/test/Gopkg.lock +++ b/test/Gopkg.lock @@ -2,7 +2,7 @@ [[projects]] - digest = "1:8b95956b70e181b19025c7ba3578fdfd8efbec4ce916490700488afb9218972c" + digest = "1:f28b5363f59b4c568f37ebaecf1d739d0dc5818f57b07f9bda776367ffe046fb" name = "cloud.google.com/go" packages = [ "compute/metadata", @@ -13,12 +13,12 @@ "internal/version", "storage", ] - pruneopts = "" - revision = "64a2037ec6be8a4b0c1d1f706ed35b428b989239" - version = "v0.26.0" + pruneopts = "UT" + revision = "28a4bc8c44b3acbcc482cff0cdf7de29a4688b61" + version = "v0.35.1" [[projects]] - digest = "1:4d581a0828b5c0f9e252edb5cd85f2c5c20809783dddedae5be94cc8ee26639b" + digest = "1:f0ca0b3256cfe939f5b5d7fb9980d3805f0a9c0d7e2ce5c990112627e77018d7" name = "github.com/aws/aws-sdk-go" packages = [ "aws", @@ -70,67 +70,86 @@ "service/sqs", "service/sts", ] - pruneopts = "" + pruneopts = "UT" revision = "3afa899153ab972b5e6bf2552721b677e3fdaeb5" version = "v1.16.20" [[projects]] - digest = "1:b529f4bf748979caa18b599d40d13e8b6e591a74b340f315ce4f95e119c288c2" + digest = "1:7b94d37d65c0445053c6f3e73090e3966c1c29127035492c349e14f25c440359" name = "github.com/boombuler/barcode" packages = [ ".", "qr", "utils", ] - pruneopts = "" + pruneopts = "UT" revision = "3cfea5ab600ae37946be2b763b8ec2c1cf2d272d" version = "v1.0.0" [[projects]] - digest = "1:0deddd908b6b4b768cfc272c16ee61e7088a60f7fe2f06c547bd3d8e1f8b8e77" + digest = "1:ffe9824d294da03b391f44e1ae8281281b4afc1bdaa9588c9097785e3af10cec" name = "github.com/davecgh/go-spew" packages = ["spew"] - pruneopts = "" + pruneopts = "UT" revision = "8991bc29aa16c548c550c7ff78260e27b9ab7c73" version = "v1.1.1" [[projects]] - digest = "1:b13707423743d41665fd23f0c36b2f37bb49c30e94adb813319c44188a51ba22" + branch = "master" + digest = "1:ecdc8e0fe3bc7d549af1c9c36acf3820523b707d6c071b6d0c3860882c6f7b42" + name = "github.com/docker/spdystream" + packages = [ + ".", + "spdy", + ] + pruneopts = "UT" + revision = "6480d4af844c189cf5dd913db24ddd339d3a4f85" + +[[projects]] + digest = "1:2cd7915ab26ede7d95b8749e6b1f933f1c6d5398030684e6505940a10f31cfda" name = "github.com/ghodss/yaml" packages = ["."] - pruneopts = "" + pruneopts = "UT" revision = "0ca9ea5df5451ffdf184b4428c902747c2c11cd7" version = "v1.0.0" [[projects]] - digest = "1:e692d16fdfbddb94e9e4886aaf6c08bdbae5cb4ac80651445de9181b371c6e46" + branch = "master" + digest = "1:d0557c61d3acd22f04f7b6b675b847381034b229b7dce10b1d31bfa0d7f30017" + name = "github.com/go-errors/errors" + packages = ["."] + pruneopts = "UT" + revision = "d98b870cc4e05f1545532a80e9909be8216095b6" + +[[projects]] + digest = "1:ec6f9bf5e274c833c911923c9193867f3f18788c461f76f05f62bb1510e0ae65" name = "github.com/go-sql-driver/mysql" packages = ["."] - pruneopts = "" + pruneopts = "UT" revision = "72cd26f257d44c1114970e19afddcd812016007e" version = "v1.4.1" [[projects]] - digest = "1:527e1e468c5586ef2645d143e9f5fbd50b4fe5abc8b1e25d9f1c416d22d24895" + digest = "1:b402bb9a24d108a9405a6f34675091b036c8b056aac843bf6ef2389a65c5cf48" name = "github.com/gogo/protobuf" packages = [ "proto", "sortkeys", ] - pruneopts = "" + pruneopts = "UT" revision = "4cbf7e384e768b4e01799441fdf2a706a5635ae7" version = "v1.2.0" [[projects]] branch = "master" - digest = "1:107b233e45174dbab5b1324201d092ea9448e58243ab9f039e4c0f332e121e3a" + digest = "1:1ba1d79f2810270045c328ae5d674321db34e3aae468eb4233883b473c5c0467" name = "github.com/golang/glog" packages = ["."] - pruneopts = "" + pruneopts = "UT" revision = "23def4e6c14b4da8ac2ed8007337bc5eb5007998" [[projects]] - digest = "1:3dd078fda7500c341bc26cfbc6c6a34614f295a2457149fc1045cab767cbcf18" + digest = "1:5d1b5a25486fc7d4e133646d834f6fca7ba1cef9903d40e7aa786c41b89e9e91" name = "github.com/golang/protobuf" packages = [ "proto", @@ -140,70 +159,78 @@ "ptypes/duration", "ptypes/timestamp", ] - pruneopts = "" + pruneopts = "UT" revision = "aa810b61a9c79d51363740d207bb46cf8e620ed5" version = "v1.2.0" [[projects]] branch = "master" - digest = "1:1e5b1e14524ed08301977b7b8e10c719ed853cbf3f24ecb66fae783a46f207a6" + digest = "1:0bfbe13936953a98ae3cfe8ed6670d396ad81edf069a806d2f6515d7bb6950df" name = "github.com/google/btree" packages = ["."] - pruneopts = "" + pruneopts = "UT" revision = "4030bb1f1f0c35b30ca7009e9ebd06849dd45306" [[projects]] branch = "master" - digest = "1:754f77e9c839b24778a4b64422236d38515301d2baeb63113aa3edc42e6af692" + digest = "1:3ee90c0d94da31b442dde97c99635aaafec68d0b8a3c12ee2075c6bdabeec6bb" name = "github.com/google/gofuzz" packages = ["."] - pruneopts = "" + pruneopts = "UT" revision = "24818f796faf91cd76ec7bddd72458fbced7a6c1" [[projects]] - digest = "1:c1d7e883c50a26ea34019320d8ae40fad86c9e5d56e63a1ba2cb618cef43e986" + digest = "1:8f8811f9be822914c3a25c6a071e93beb4c805d7b026cbf298bc577bc1cc945b" name = "github.com/google/uuid" packages = ["."] - pruneopts = "" + pruneopts = "UT" revision = "064e2069ce9c359c118179501254f67d7d37ba24" version = "0.2" [[projects]] - digest = "1:55c1b46a80db2baf4d762c1d0b5cb4946e46125baa02b8959310abab15b54aee" + digest = "1:856bd1e35f6da8ce5671a5df09d0e89bf01e9b74b3dabb6d097d39b3813801e1" name = "github.com/googleapis/gax-go" - packages = [ - ".", - "v2", - ] - pruneopts = "" + packages = ["v2"] + pruneopts = "UT" revision = "c8a15bac9b9fe955bd9f900272f9a306465d28cf" version = "v2.0.3" [[projects]] - digest = "1:16b2837c8b3cf045fa2cdc82af0cf78b19582701394484ae76b2c3bc3c99ad73" + digest = "1:65c4414eeb350c47b8de71110150d0ea8a281835b1f386eacaa3ad7325929c21" name = "github.com/googleapis/gnostic" packages = [ "OpenAPIv2", "compiler", "extensions", ] - pruneopts = "" + pruneopts = "UT" revision = "7c663266750e7d82587642f65e60bc4083f1f84e" version = "v0.2.0" [[projects]] branch = "master" - digest = "1:5e345eb75d8bfb2b91cfbfe02a82a79c0b2ea55cf06c5a4d180a9321f36973b4" + digest = "1:86c1210529e69d69860f2bb3ee9ccce0b595aa3f9165e7dd1388e5c612915888" name = "github.com/gregjones/httpcache" packages = [ ".", "diskcache", ] - pruneopts = "" + pruneopts = "UT" revision = "c63ab54fda8f77302f8d414e19933f2b6026a089" [[projects]] - digest = "1:f095be8f8054d5dfca75318f21aabbad9b88862f6dfadad7f7cfddb1ec8be9cf" + digest = "1:ca55309eddc07bc8324b7f76c9b7d42b08adc87ca73018fc95536c255958aa2d" + name = "github.com/gruntwork-io/gruntwork-cli" + packages = [ + "collections", + "errors", + ] + pruneopts = "UT" + revision = "6a2163138f3d10377f313428e7e367b0a6c0c1c9" + version = "v0.4.2" + +[[projects]] + digest = "1:12cf87df04f335d94d1346edd77536ca505a4dda0afd5137150cd440e5c2d32c" name = "github.com/gruntwork-io/terratest" packages = [ "modules/aws", @@ -212,6 +239,8 @@ "modules/environment", "modules/files", "modules/gcp", + "modules/helm", + "modules/http-helper", "modules/k8s", "modules/logger", "modules/packer", @@ -222,46 +251,54 @@ "modules/terraform", "modules/test-structure", ] - pruneopts = "" - revision = "913f60214f225bac27ef03ee2b93a92931bc9786" - version = "v0.13.23" + pruneopts = "UT" + revision = "b6077c8c1ecfd290ee1f37e1e51472a34d30cb38" + version = "v0.13.29" [[projects]] - digest = "1:31bfd110d31505e9ffbc9478e31773bf05bf02adcaeb9b139af42684f9294c13" + digest = "1:a0cefd27d12712af4b5018dc7046f245e1e3b5760e2e848c30b171b570708f9b" name = "github.com/imdario/mergo" packages = ["."] - pruneopts = "" + pruneopts = "UT" revision = "7c29201646fa3de8506f701213473dd407f19646" version = "v0.3.7" [[projects]] - digest = "1:13fe471d0ed891e8544eddfeeb0471fd3c9f2015609a1c000aefdedf52a19d40" + digest = "1:870d441fe217b8e689d7949fef6e43efbc787e50f200cb1e70dbca9204a1d6be" + name = "github.com/inconshreveable/mousetrap" + packages = ["."] + pruneopts = "UT" + revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75" + version = "v1.0" + +[[projects]] + digest = "1:bb81097a5b62634f3e9fec1014657855610c82d19b9a40c17612e32651e35dca" name = "github.com/jmespath/go-jmespath" packages = ["."] - pruneopts = "" + pruneopts = "UT" revision = "c2b33e84" [[projects]] - digest = "1:b79fc583e4dc7055ed86742e22164ac41bf8c0940722dbcb600f1a3ace1a8cb5" + digest = "1:3e551bbb3a7c0ab2a2bf4660e7fcad16db089fdcfbb44b0199e62838038623ea" name = "github.com/json-iterator/go" packages = ["."] - pruneopts = "" + pruneopts = "UT" revision = "1624edc4454b8682399def8740d46db5e4362ba4" version = "v1.1.5" [[projects]] - digest = "1:096a8a9182648da3d00ff243b88407838902b6703fc12657f76890e08d1899bf" + digest = "1:78bbb1ba5b7c3f2ed0ea1eab57bdd3859aec7e177811563edc41198a760b06af" name = "github.com/mitchellh/go-homedir" packages = ["."] - pruneopts = "" + pruneopts = "UT" revision = "ae18d6b8b3205b561c79e8e5f69bff09736185f4" version = "v1.0.0" [[projects]] - digest = "1:0c0ff2a89c1bb0d01887e1dac043ad7efbf3ec77482ef058ac423d13497e16fd" + digest = "1:33422d238f147d247752996a26574ac48dcf472976eda7f5134015f06bf16563" name = "github.com/modern-go/concurrent" packages = ["."] - pruneopts = "" + pruneopts = "UT" revision = "bacd9c7ef1dd9b15be4a9909b8ac7a4e313eec94" version = "1.0.3" @@ -269,67 +306,83 @@ digest = "1:e32bdbdb7c377a07a9a46378290059822efdce5c8d96fe71940d87cb4f918855" name = "github.com/modern-go/reflect2" packages = ["."] - pruneopts = "" + pruneopts = "UT" revision = "4b7aa43c6742a2c18fdef89dd197aaae7dac7ccd" version = "1.0.1" [[projects]] branch = "master" - digest = "1:c24598ffeadd2762552269271b3b1510df2d83ee6696c1e543a0ff653af494bc" + digest = "1:3bf17a6e6eaa6ad24152148a631d18662f7212e21637c2699bff3369b7f00fa2" name = "github.com/petar/GoLLRB" packages = ["llrb"] - pruneopts = "" + pruneopts = "UT" revision = "53be0d36a84c2a886ca057d34b6aa4468df9ccb4" [[projects]] - digest = "1:b46305723171710475f2dd37547edd57b67b9de9f2a6267cafdd98331fd6897f" + digest = "1:0e7775ebbcf00d8dd28ac663614af924411c868dca3d5aa762af0fae3808d852" name = "github.com/peterbourgon/diskv" packages = ["."] - pruneopts = "" + pruneopts = "UT" revision = "5f041e8faa004a95c88a202771f4cc3e991971e6" version = "v2.0.1" [[projects]] - digest = "1:256484dbbcd271f9ecebc6795b2df8cad4c458dd0f5fd82a8c2fa0c29f233411" + digest = "1:0028cb19b2e4c3112225cd871870f2d9cf49b9b4276531f03438a88e94be86fe" name = "github.com/pmezard/go-difflib" packages = ["difflib"] - pruneopts = "" + pruneopts = "UT" revision = "792786c7400a136282c1664665ae0a8db921c6c2" version = "v1.0.0" [[projects]] - digest = "1:d8c398c75a666d415196ba289402c3f52b595d8cede451a9e57278354e327a93" + digest = "1:dc56e7178a82f8e84960ed9efa7d4ebd0a6d7864894bfc46f8a35ee5eae7153a" name = "github.com/pquerna/otp" packages = [ ".", "hotp", "totp", ] - pruneopts = "" + pruneopts = "UT" revision = "be78767b3e392ce45ea73444451022a6fc32ad0d" version = "v1.1.0" [[projects]] - digest = "1:cbaf13cdbfef0e4734ed8a7504f57fe893d471d62a35b982bf6fb3f036449a66" + digest = "1:645cabccbb4fa8aab25a956cbcbdf6a6845ca736b2c64e197ca7cbb9d210b939" + name = "github.com/spf13/cobra" + packages = ["."] + pruneopts = "UT" + revision = "ef82de70bb3f60c65fb8eebacbb2d122ef517385" + version = "v0.0.3" + +[[projects]] + digest = "1:c1b1102241e7f645bc8e0c22ae352e8f0dc6484b6cb4d132fa9f24174e0119e2" name = "github.com/spf13/pflag" packages = ["."] - pruneopts = "" + pruneopts = "UT" revision = "298182f68c66c05229eb03ac171abe6e309ee79a" version = "v1.0.3" [[projects]] - digest = "1:381bcbeb112a51493d9d998bbba207a529c73dbb49b3fd789e48c63fac1f192c" + digest = "1:5da8ce674952566deae4dbc23d07c85caafc6cfa815b0b3e03e41979cedb8750" name = "github.com/stretchr/testify" packages = [ "assert", "require", ] - pruneopts = "" + pruneopts = "UT" revision = "ffdc059bfe9ce6a4e144ba849dbedead332c6053" version = "v1.3.0" [[projects]] - digest = "1:b1bb9332f6cb7821a730e1681819b2813340eba5e873f05381815a7c6807d172" + digest = "1:b24d38b282bacf9791408a080f606370efa3d364e4b5fd9ba0f7b87786d3b679" + name = "github.com/urfave/cli" + packages = ["."] + pruneopts = "UT" + revision = "cfb38830724cc34fedffe9a2a29fb54fa9169cd1" + version = "v1.20.0" + +[[projects]] + digest = "1:785adacba41666eb1de74447b5538476b9a9da178bd7d15ba3aeb2879ed07ed3" name = "go.opencensus.io" packages = [ ".", @@ -347,13 +400,13 @@ "trace/propagation", "trace/tracestate", ] - pruneopts = "" + pruneopts = "UT" revision = "2b5032d79456124f42db6b7eb19ac6c155449dc2" version = "v0.19.0" [[projects]] branch = "master" - digest = "1:59b49c47c11a48f1054529207f65907c014ecf5f9a7c0d9c0f1616dec7b062ed" + digest = "1:30e9f5bea4df0d1a573ed89a85cc680ab05dfc078f6a21e627db236f29650a11" name = "golang.org/x/crypto" packages = [ "curve25519", @@ -366,12 +419,12 @@ "ssh/agent", "ssh/terminal", ] - pruneopts = "" + pruneopts = "UT" revision = "ff983b9c42bc9fbf91556e191cc8efb585c16908" [[projects]] branch = "master" - digest = "1:7ec13687f85b25087fe05f6ea8dd116013a8263f8eb7e057da7664bc7599d2d4" + digest = "1:470efb06ada11351d90ee09868d84c622cc949a59c165b99d33d555dacbde74b" name = "golang.org/x/net" packages = [ "context", @@ -383,11 +436,11 @@ "internal/timeseries", "trace", ] - pruneopts = "" + pruneopts = "UT" revision = "915654e7eabcea33ae277abbecf52f0d8b7a9fdc" [[projects]] - digest = "1:b697592485cb412be4188c08ca0beed9aab87f36b86418e21acc4a3998f63734" + digest = "1:f645667d687fc8bf228865a2c5455824ef05bad08841e673673ef2bb89ac5b90" name = "golang.org/x/oauth2" packages = [ ".", @@ -396,22 +449,22 @@ "jws", "jwt", ] - pruneopts = "" + pruneopts = "UT" revision = "d2e6202438beef2727060aa7cabdd924d92ebfd9" [[projects]] branch = "master" - digest = "1:7e3b61f51ebcb58b3894928ed7c63aae68820dec1dd57166e5d6e65ef2868f40" + digest = "1:bb644db32f5bc1e327a0f748ec871edc4dc46f19b4fbbc15bfd95b1460646537" name = "golang.org/x/sys" packages = [ "unix", "windows", ] - pruneopts = "" + pruneopts = "UT" revision = "b90733256f2e882e81d52f9126de08df5615afd9" [[projects]] - digest = "1:5acd3512b047305d49e8763eef7ba423901e85d5dd2fd1e71778a0ea8de10bd4" + digest = "1:a2ab62866c75542dd18d2b069fec854577a20211d7c0ea6ae746072a1dccdd18" name = "golang.org/x/text" packages = [ "collate", @@ -429,21 +482,21 @@ "unicode/norm", "unicode/rangetable", ] - pruneopts = "" + pruneopts = "UT" revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" version = "v0.3.0" [[projects]] branch = "master" - digest = "1:14cb1d4240bcbbf1386ae763957e04e2765ec4e4ce7bb2769d05fa6faccd774e" + digest = "1:9fdc2b55e8e0fafe4b41884091e51e77344f7dc511c5acedcfd98200003bff90" name = "golang.org/x/time" packages = ["rate"] - pruneopts = "" + pruneopts = "UT" revision = "85acf8d2951cb2a3bde7632f9ff273ef0379bcbd" [[projects]] branch = "master" - digest = "1:a0b98ef68cfcc79279eb28a8d27d0f3b03045375bdf12cca49d27a1996442b8e" + digest = "1:d586ebbd814cf4a680afbac35457f94a4ec9ae8305d9831b7a290c32baa0bb31" name = "google.golang.org/api" packages = [ "compute/v1", @@ -458,11 +511,11 @@ "transport/http", "transport/http/internal/propagation", ] - pruneopts = "" + pruneopts = "UT" revision = "59f8eae6bc597bfd5d5b292134c4b19d8955ed8a" [[projects]] - digest = "1:bc09e719c4e2a15d17163f5272d9a3131c45d77542b7fdc53ff518815bc19ab3" + digest = "1:7bc25c2efff76b31f146caf630c617be9b666c6164f0632050466fbec0500125" name = "google.golang.org/appengine" packages = [ ".", @@ -477,13 +530,13 @@ "internal/urlfetch", "urlfetch", ] - pruneopts = "" + pruneopts = "UT" revision = "e9657d882bb81064595ca3b56cbe2546bbabf7b1" version = "v1.4.0" [[projects]] branch = "master" - digest = "1:2255fee1df9431ec9c0a373a4e112a60b28ae435d7af57d69f4f2e2e86800cad" + digest = "1:a7d48ca460ca1b4f6ccd8c95502443afa05df88aee84de7dbeb667a8754e8fa6" name = "google.golang.org/genproto" packages = [ "googleapis/api/annotations", @@ -491,11 +544,11 @@ "googleapis/rpc/code", "googleapis/rpc/status", ] - pruneopts = "" + pruneopts = "UT" revision = "8819c946db4494a2259bf100a377f51aa585d893" [[projects]] - digest = "1:39d4d828b87d58d114fdc211f0638f32dcae84019fe17d6b48f9b697f4b60213" + digest = "1:9ab5a33d8cb5c120602a34d2e985ce17956a4e8c2edce7e6961568f95e40c09a" name = "google.golang.org/grpc" packages = [ ".", @@ -530,29 +583,29 @@ "status", "tap", ] - pruneopts = "" + pruneopts = "UT" revision = "a02b0774206b209466313a0b525d2c738fe407eb" version = "v1.18.0" [[projects]] - digest = "1:75fb3fcfc73a8c723efde7777b40e8e8ff9babf30d8c56160d01beffea8a95a6" + digest = "1:2d1fbdc6777e5408cabeb02bf336305e724b925ff4546ded0fa8715a7267922a" name = "gopkg.in/inf.v0" packages = ["."] - pruneopts = "" + pruneopts = "UT" revision = "d2d2541c53f18d2a059457998ce2876cc8e67cbf" version = "v0.9.1" [[projects]] - digest = "1:cedccf16b71e86db87a24f8d4c70b0a855872eb967cb906a66b95de56aefbd0d" + digest = "1:4d2e5a73dc1500038e504a8d78b986630e3626dc027bc030ba5c75da257cdb96" name = "gopkg.in/yaml.v2" packages = ["."] - pruneopts = "" + pruneopts = "UT" revision = "51d6538a90f86fe93ac480b35f37b2be17fef232" version = "v2.2.2" [[projects]] branch = "master" - digest = "1:40754e07d7875ab0b4480e711dcd0ba1274deaf1e676096834bb757d3333a04a" + digest = "1:5d9a48e3144dfc06d3c5218b38164102f62300e83ffdd9a2d87b3c0b4135612f" name = "k8s.io/api" packages = [ "admissionregistration/v1alpha1", @@ -587,12 +640,12 @@ "storage/v1alpha1", "storage/v1beta1", ] - pruneopts = "" + pruneopts = "UT" revision = "173ce66c1e39d1d0f56e0b3347ff2988068aecd0" [[projects]] branch = "release-1.12" - digest = "1:8cac653dd97e89700e00ce6ea7134cf28b3e1b59615ab99464760e230fab9860" + digest = "1:f732c584e8a48a368d35289823fbb450987c5b242f1b96e1cae39774b2c1215c" name = "k8s.io/apimachinery" packages = [ "pkg/api/errors", @@ -618,6 +671,8 @@ "pkg/util/clock", "pkg/util/errors", "pkg/util/framer", + "pkg/util/httpstream", + "pkg/util/httpstream/spdy", "pkg/util/intstr", "pkg/util/json", "pkg/util/naming", @@ -629,14 +684,15 @@ "pkg/util/yaml", "pkg/version", "pkg/watch", + "third_party/forked/golang/netutil", "third_party/forked/golang/reflect", ] - pruneopts = "" + pruneopts = "UT" revision = "49ce2735e5074ffc3f8190c8406cf51a96302dad" [[projects]] branch = "release-9.0" - digest = "1:203ee3d901467a9bfa00c1859438f168288431405b7a30c7c6f14f8391cafa6f" + digest = "1:a92af87ab3cdd5289ba3b0b44c3e62051ba532ebb5b4ee3f6da7adf9e1a64169" name = "k8s.io/client-go" packages = [ "discovery", @@ -688,8 +744,10 @@ "tools/clientcmd/api/latest", "tools/clientcmd/api/v1", "tools/metrics", + "tools/portforward", "tools/reference", "transport", + "transport/spdy", "util/cert", "util/connrotation", "util/flowcontrol", @@ -697,14 +755,24 @@ "util/integer", "util/jsonpath", ] - pruneopts = "" + pruneopts = "UT" revision = "701b913670036511e3d752318272c97f1a2a2edd" +[[projects]] + digest = "1:51314886250b22f73377febaadd972084684122e2e1d3004f388dff60754ba05" + name = "k8s.io/kubernetes" + packages = ["pkg/kubectl/generate"] + pruneopts = "UT" + revision = "721bfa751924da8d1680787490c54b9179b1fed0" + version = "v1.13.3" + [solve-meta] analyzer-name = "dep" analyzer-version = 1 input-imports = [ "github.com/gruntwork-io/terratest/modules/gcp", + "github.com/gruntwork-io/terratest/modules/helm", + "github.com/gruntwork-io/terratest/modules/http-helper", "github.com/gruntwork-io/terratest/modules/k8s", "github.com/gruntwork-io/terratest/modules/logger", "github.com/gruntwork-io/terratest/modules/random", diff --git a/test/Gopkg.toml b/test/Gopkg.toml index 1a2445c..2436a79 100644 --- a/test/Gopkg.toml +++ b/test/Gopkg.toml @@ -23,7 +23,7 @@ [[constraint]] name = "github.com/gruntwork-io/terratest" - version = "0.13.23" + version = "0.13.28" [prune] go-tests = true From a7e672d3470f394046fe023f7e7c3f929d1a8ea7 Mon Sep 17 00:00:00 2001 From: Rob Morgan Date: Tue, 26 Feb 2019 15:27:13 +0100 Subject: [PATCH 10/15] clarify workaround better --- examples/gke-basic-tiller/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/gke-basic-tiller/main.tf b/examples/gke-basic-tiller/main.tf index ca2c862..212cdce 100644 --- a/examples/gke-basic-tiller/main.tf +++ b/examples/gke-basic-tiller/main.tf @@ -167,7 +167,7 @@ resource "kubernetes_cluster_role_binding" "user" { subject { # this is a workaround for https://github.com/terraform-providers/terraform-provider-kubernetes/issues/204. - # we have to set an empty api_group or the k8s call will fail. + # we have to set an empty api_group or the k8s call will fail. It will be fixed in v1.5.2 of the k8s provider. api_group = "" kind = "ServiceAccount" From cbc31c0c4631b341d88e8724496be38f2fd9c87d Mon Sep 17 00:00:00 2001 From: Rob Morgan Date: Tue, 26 Feb 2019 15:29:16 +0100 Subject: [PATCH 11/15] update tests to install nginx using helm --- test/charts/minimal-pod/.helmignore | 21 +++++ test/charts/minimal-pod/Chart.yaml | 5 ++ .../charts/minimal-pod/templates/_helpers.tpl | 32 ++++++++ test/charts/minimal-pod/templates/pod.yaml | 18 +++++ test/charts/minimal-pod/values.yaml | 1 + test/gke_basic_tiller_test.go | 81 +++++++++++++++++-- test/gke_cluster_test.go | 2 + 7 files changed, 154 insertions(+), 6 deletions(-) create mode 100644 test/charts/minimal-pod/.helmignore create mode 100644 test/charts/minimal-pod/Chart.yaml create mode 100644 test/charts/minimal-pod/templates/_helpers.tpl create mode 100644 test/charts/minimal-pod/templates/pod.yaml create mode 100644 test/charts/minimal-pod/values.yaml diff --git a/test/charts/minimal-pod/.helmignore b/test/charts/minimal-pod/.helmignore new file mode 100644 index 0000000..f0c1319 --- /dev/null +++ b/test/charts/minimal-pod/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/test/charts/minimal-pod/Chart.yaml b/test/charts/minimal-pod/Chart.yaml new file mode 100644 index 0000000..9b6289f --- /dev/null +++ b/test/charts/minimal-pod/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: minimal-pod +version: 0.1.0 diff --git a/test/charts/minimal-pod/templates/_helpers.tpl b/test/charts/minimal-pod/templates/_helpers.tpl new file mode 100644 index 0000000..3e013e6 --- /dev/null +++ b/test/charts/minimal-pod/templates/_helpers.tpl @@ -0,0 +1,32 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "minimal-pod.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "minimal-pod.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "minimal-pod.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/test/charts/minimal-pod/templates/pod.yaml b/test/charts/minimal-pod/templates/pod.yaml new file mode 100644 index 0000000..6d84a9a --- /dev/null +++ b/test/charts/minimal-pod/templates/pod.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Pod +metadata: + name: {{ include "minimal-pod.fullname" . }} + labels: + app.kubernetes.io/name: {{ include "minimal-pod.name" . }} + helm.sh/chart: {{ include "minimal-pod.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image }}" + ports: + - name: http + containerPort: 80 + protocol: TCP + diff --git a/test/charts/minimal-pod/values.yaml b/test/charts/minimal-pod/values.yaml new file mode 100644 index 0000000..c3a88d0 --- /dev/null +++ b/test/charts/minimal-pod/values.yaml @@ -0,0 +1 @@ +image: "" diff --git a/test/gke_basic_tiller_test.go b/test/gke_basic_tiller_test.go index 7fbf7c9..863bd9d 100644 --- a/test/gke_basic_tiller_test.go +++ b/test/gke_basic_tiller_test.go @@ -1,10 +1,16 @@ package test import ( + "fmt" "path/filepath" + "strings" "testing" + "time" "github.com/gruntwork-io/terratest/modules/gcp" + "github.com/gruntwork-io/terratest/modules/helm" + "github.com/gruntwork-io/terratest/modules/http-helper" + "github.com/gruntwork-io/terratest/modules/k8s" "github.com/gruntwork-io/terratest/modules/logger" "github.com/gruntwork-io/terratest/modules/random" "github.com/gruntwork-io/terratest/modules/terraform" @@ -12,6 +18,8 @@ import ( ) func TestGKEBasicTiller(t *testing.T) { + //t.Parallel() + // Uncomment any of the following to skip that section during the test // os.Setenv("SKIP_create_test_copy_of_examples", "true") // os.Setenv("SKIP_create_terratest_options", "true") @@ -20,12 +28,6 @@ func TestGKEBasicTiller(t *testing.T) { // os.Setenv("SKIP_wait_for_workers", "true") // os.Setenv("SKIP_cleanup", "true") - // - [x] Create GKE cluster - // - [x] Configure kubectl - // - [x] Securely install Helm using terraform-kubernetes-helm - // - [ ] Install an example chart (nginx?) - // - [ ] test port 80 open/running - // Create a directory path that won't conflict workingDir := filepath.Join(".", "stages", t.Name()) @@ -63,4 +65,71 @@ func TestGKEBasicTiller(t *testing.T) { test_structure.RunTestStage(t, "wait_for_workers", func() { verifyGkeNodesAreReady(t) }) + + test_structure.RunTestStage(t, "helm_install", func() { + // Path to the helm chart we will test + helmChartPath := "charts/minimal-pod" + + // Setup the kubectl config and context. Here we choose to use the defaults, which is: + // - HOME/.kube/config for the kubectl config file + // - Current context of the kubectl config file + // We also specify that we are working in the default namespace (required to get the Pod) + kubectlOptions := k8s.NewKubectlOptions("", "") + kubectlOptions.Namespace = "default" + + // We generate a unique release name so that we can refer to after deployment. + // By doing so, we can schedule the delete call here so that at the end of the test, we run + // `helm delete RELEASE_NAME` to clean up any resources that were created. + releaseName := fmt.Sprintf("nginx-%s", strings.ToLower(random.UniqueId())) + + // Setup the args. For this test, we will set the following input values: + // - image=nginx:1.15.8 + // - fullnameOverride=minimal-pod-RANDOM_STRING + // We use a fullnameOverride so we can find the Pod later during verification + podName := fmt.Sprintf("%s-minimal-pod", releaseName) + options := &helm.Options{ + SetValues: map[string]string{ + "image": "nginx:1.15.8", + "fullnameOverride": podName, + }, + EnvVars: map[string]string{ + "HELM_TLS_VERIFY": "true", + "HELM_TLS_ENABLE": "true", + }, + } + + // Deploy the chart using `helm install`. Note that we use the version without `E`, since we want to assert the + // install succeeds without any errors. + helm.Install(t, options, helmChartPath, releaseName) + + // Now that the chart is deployed, verify the deployment. This function will open a tunnel to the Pod and hit the + // nginx container endpoint. + verifyNginxPod(t, kubectlOptions, podName) + }) +} + +// verifyNginxPod will open a tunnel to the Pod and hit the endpoint to verify the nginx welcome page is shown. +func verifyNginxPod(t *testing.T, kubectlOptions *k8s.KubectlOptions, podName string) { + // Wait for the pod to come up. It takes some time for the Pod to start, so retry a few times. + retries := 15 + sleep := 5 * time.Second + k8s.WaitUntilPodAvailable(t, kubectlOptions, podName, retries, sleep) + + // We will first open a tunnel to the pod, making sure to close it at the end of the test. + tunnel := k8s.NewTunnel(kubectlOptions, k8s.ResourceTypePod, podName, 0, 80) + defer tunnel.Close() + tunnel.ForwardPort(t) + + // ... and now that we have the tunnel, we will verify that we get back a 200 OK with the nginx welcome page. + // It takes some time for the Pod to start, so retry a few times. + endpoint := fmt.Sprintf("http://%s", tunnel.Endpoint()) + http_helper.HttpGetWithRetryWithCustomValidation( + t, + endpoint, + retries, + sleep, + func(statusCode int, body string) bool { + return statusCode == 200 && strings.Contains(body, "Welcome to nginx") + }, + ) } diff --git a/test/gke_cluster_test.go b/test/gke_cluster_test.go index ae5f3b6..b2c2395 100644 --- a/test/gke_cluster_test.go +++ b/test/gke_cluster_test.go @@ -13,6 +13,8 @@ import ( ) func TestGKECluster(t *testing.T) { + //t.Parallel() + // Uncomment any of the following to skip that section during the test // os.Setenv("SKIP_create_test_copy_of_examples", "true") // os.Setenv("SKIP_create_terratest_options", "true") From 6cb83e7b1160f1e17a25e94904ed88329e140531 Mon Sep 17 00:00:00 2001 From: Rob Morgan Date: Tue, 26 Feb 2019 15:29:37 +0100 Subject: [PATCH 12/15] add WIP readme --- examples/gke-basic-tiller/README.md | 139 ++++++++++++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 examples/gke-basic-tiller/README.md diff --git a/examples/gke-basic-tiller/README.md b/examples/gke-basic-tiller/README.md new file mode 100644 index 0000000..adb3e32 --- /dev/null +++ b/examples/gke-basic-tiller/README.md @@ -0,0 +1,139 @@ +# GKE Basic Helm Example + +This example shows how to use Terraform to launch a GKE cluster with Helm configured and installed. We achieve this by calling out to our `kubergrunt` utility in order to securely deploy Tiller - the server component of Helm. + + +## Background + +We strongly recommend reading [our guide on Helm](https://github.com/gruntwork-io/kubergrunt/blob/master/HELM_GUIDE.md) +before continuing with this guide for a background on Helm, Tiller, and the security model backing it. + + +## Overview + +In this guide we will walk through the steps necessary to get up and running with deploying Tiller on GKE using this +module. Here are the steps: + +1. [Install the necessary tools](#installing-necessary-tools) +1. [Apply the Terraform code](#apply-the-terraform-code) +1. [Verify the deployment](#verify-tiller-deployment) +1. [Granting access to additional roles](#granting-access-to-additional-users) +1. [Upgrading the deployed Tiller instance](#upgrading-deployed-tiller) + +## Installing necessary tools + +In addition to `terraform`, this guide relies on the `gcloud` and `kubectl` tools to manage the cluster. In addition +we use `kubergrunt` to manage the deployment of Tiller. You can read more about the decision behind this approach in +[the Appendix](#appendix-a-why-kubergrunt) of this guide. + +This means that your system needs to be configured to be able to find `terraform`, `gcloud`, `kubectl`, `kubergrunt`, +and `helm` client utilities on the system `PATH`. Here are the installation guide for each tool: + +1. [`gcloud`](https://cloud.google.com/sdk/gcloud/) +1. [`kubectl`](https://kubernetes.io/docs/tasks/tools/install-kubectl/) +1. [`terraform`](https://learn.hashicorp.com/terraform/getting-started/install.html) +1. [`helm` client](https://docs.helm.sh/using_helm/#installing-helm) +1. [`kubergrunt`](https://github.com/gruntwork-io/kubergrunt#installation) + +Make sure the binaries are discoverable in your `PATH` variable. See [this Stack Overflow +post](https://stackoverflow.com/questions/14637979/how-to-permanently-set-path-on-linux-unix) for instructions on +setting up your `PATH` on Unix, and [this +post](https://stackoverflow.com/questions/1618280/where-can-i-set-path-to-make-exe-on-windows) for instructions on +Windows. + +## Apply the Terraform Code + +Now that all the prerequisite tools are installed, we are ready to deploy the GKE cluster with Tiller installed! + +1. If you haven't already, clone this repo: + - `git clone https://github.com/gruntwork-io/terraform-google-gke.git` +1. Make sure you are in the `gke-basic-tiller` example folder: + - `cd examples/gke-basic-tiller` +1. Initialize terraform: + - `terraform init` +1. Check the terraform plan: + - `terraform plan` +1. Apply the terraform code: + - `terraform apply` + - Fill in the required variables based on your needs. + +**Note:** For simplicity this example uses `kubergrunt` to install Tiller into the `kube-system` namespace. However in +a production deployment we strongly recommend you segregate the Tiller resources into a separate namespace. + +As part of the deployment, `kubergrunt` will: + +- Create a new TLS certificate key pair to use as the CA and upload it to Kubernetes as a `Secret` in the `kube-system` + namespace. +- Using the generated CA TLS certificate key pair, create a signed TLS certificate key pair to use to identify the + Tiller server and upload it to Kubernetes as a `Secret` in `kube-system`. +- Deploy Tiller with the following configurations turned on: + - TLS verification + - `Secrets` as the storage engine + - Provisioned in the `kube-system` namespace using the `default` service account. + +- Grant access to the provided RBAC entity and configure the local helm client to use those credentials: + - Using the CA TLS certificate key pair, create a signed TLS certificate key pair to use to identify the client. + - Upload the certificate key pair to the `kube-system`. + - Grant the RBAC entity access to: + - Get the client certificate `Secret` (`kubergrunt helm configure` uses this to install the client certificate + key pair locally) + - Get and List pods in `kube-system` namespace (the `helm` client uses this to find the Tiller pod) + - Create a port forward to the Tiller pod (the `helm` client uses this to make requests to the Tiller pod) + + - Install the client certificate key pair to the helm home directory so the client can use it. + +You should now have a working Tiller deployment with your helm client configured to access it. +So let's verify that in the next step! + +## Verify Tiller Deployment + +To start using `helm` with the configured credentials, you need to specify the following things: + +- enable TLS verification +- use TLS credentials to authenticate +- the namespace where Tiller is deployed + +These are specified through command line arguments. If everything is configured correctly, you should be able to access +the Tiller that was deployed with the following args: + +``` +helm --tls --tls-verify --tiller-namespace NAMESPACE_OF_TILLER version +``` + +If you have access to Tiller, this should return you both the client version and the server version of Helm. + +Note that you need to pass the above CLI argument every time you want to use `helm`. This can be cumbersome, so +`kubergrunt` installs an environment file into your helm home directory that you can dot source to set environment +variables that guide `helm` to use those options: + +``` +. ~/.helm/env +helm version +``` + +## Appendix A: Why kubergrunt? + +This Terraform example is not idiomatic Terraform code in that it relies on an external binary, `kubergrunt` as opposed +to implementing the functionalities using pure Terraform providers. This approach has some noticeable drawbacks: + +- You have to install extra tools to use, so it is not a minimal `terraform init && terraform apply`. +- Portability concerns to setup, as there is no guarantee the tools work cross platform. We make every effort to test + across the major operating systems (Linux, Mac OSX, and Windows), but we can't possibly test every combination and so + there are bound to be portability issues. +- You don't have the declarative Terraform features that you come to love, such as `plan`, updates through `apply`, and + `destroy`. + +That said, we decided to use this approach because of limitations in the existing providers to implement the +functionalities here in pure Terraform code: + +- The Helm provider does not have [a resource that manages + Tiller](https://github.com/terraform-providers/terraform-provider-helm/issues/134). +- The [TLS provider](https://www.terraform.io/docs/providers/tls/index.html) stores the certificate key pairs in plain + text into the Terraform state. +- The Kubernetes Secret resource in the provider [also stores the value in plain text in the Terraform + state](https://www.terraform.io/docs/providers/kubernetes/r/secret.html). +- The grant and configure workflows are better suited as CLI tools than in Terraform. + +Note that [we intend to implement a pure Terraform version of this when the Helm provider is +updated](https://github.com/gruntwork-io/terraform-kubernetes-helm/issues/13), but we plan to continue to maintain the +`kubergrunt` approach for folks who are wary of leaking secrets into Terraform state. From 476118416acf945062085063c397138f906a7ed2 Mon Sep 17 00:00:00 2001 From: Yoriyasu Yano <430092+yorinasub17@users.noreply.github.com> Date: Thu, 28 Feb 2019 11:13:46 +0100 Subject: [PATCH 13/15] Update examples/gke-basic-tiller/dependencies.tf Co-Authored-By: robmorgan --- examples/gke-basic-tiller/dependencies.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/gke-basic-tiller/dependencies.tf b/examples/gke-basic-tiller/dependencies.tf index 689dc72..0939a3f 100644 --- a/examples/gke-basic-tiller/dependencies.tf +++ b/examples/gke-basic-tiller/dependencies.tf @@ -1,5 +1,5 @@ # --------------------------------------------------------------------------------------------------------------------- -# INTERPOLATE AND CONSTRUCT COMMAND ARGUMENTS +# INTERPOLATE AND CONSTRUCT KUBERGRUNT HELM DEPLOY COMMAND ARGUMENTS # --------------------------------------------------------------------------------------------------------------------- locals { From 8da6362273b5812976a4473f9837dd7b3f086edd Mon Sep 17 00:00:00 2001 From: Rob Morgan Date: Thu, 28 Feb 2019 11:19:20 +0100 Subject: [PATCH 14/15] clarifying why test parallelism is disabled --- test/gke_basic_tiller_test.go | 4 ++++ test/gke_cluster_test.go | 3 +++ 2 files changed, 7 insertions(+) diff --git a/test/gke_basic_tiller_test.go b/test/gke_basic_tiller_test.go index 863bd9d..89258f3 100644 --- a/test/gke_basic_tiller_test.go +++ b/test/gke_basic_tiller_test.go @@ -18,6 +18,9 @@ import ( ) func TestGKEBasicTiller(t *testing.T) { + // We are temporarily stopping the tests from running in parallel due to conflicting + // kubectl configs. This is a limitation in the current Terratest functions and will + // be fixed in a later release. //t.Parallel() // Uncomment any of the following to skip that section during the test @@ -26,6 +29,7 @@ func TestGKEBasicTiller(t *testing.T) { // os.Setenv("SKIP_terraform_apply", "true") // os.Setenv("SKIP_configure_kubectl", "true") // os.Setenv("SKIP_wait_for_workers", "true") + // os.Setenv("SKIP_helm_install", "true") // os.Setenv("SKIP_cleanup", "true") // Create a directory path that won't conflict diff --git a/test/gke_cluster_test.go b/test/gke_cluster_test.go index b2c2395..8821029 100644 --- a/test/gke_cluster_test.go +++ b/test/gke_cluster_test.go @@ -13,6 +13,9 @@ import ( ) func TestGKECluster(t *testing.T) { + // We are temporarily stopping the tests from running in parallel due to conflicting + // kubectl configs. This is a limitation in the current Terratest functions and will + // be fixed in a later release. //t.Parallel() // Uncomment any of the following to skip that section during the test From e02e546fd62a629a5f4d9a47c9af5ab109e534aa Mon Sep 17 00:00:00 2001 From: Rob Morgan Date: Thu, 28 Feb 2019 12:29:49 +0100 Subject: [PATCH 15/15] lock google providers to 2.0.0 --- examples/gke-basic-tiller/main.tf | 7 +++++++ examples/gke-regional-public-cluster/main.tf | 17 ++++++++++++----- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/examples/gke-basic-tiller/main.tf b/examples/gke-basic-tiller/main.tf index 212cdce..990ddb6 100644 --- a/examples/gke-basic-tiller/main.tf +++ b/examples/gke-basic-tiller/main.tf @@ -10,7 +10,14 @@ terraform { required_version = ">= 0.10.3" } +provider "google" { + version = "~> 2.0.0" + project = "${var.project}" + region = "${var.region}" +} + provider "google-beta" { + version = "~> 2.0.0" project = "${var.project}" region = "${var.region}" } diff --git a/examples/gke-regional-public-cluster/main.tf b/examples/gke-regional-public-cluster/main.tf index e50124c..5be1296 100644 --- a/examples/gke-regional-public-cluster/main.tf +++ b/examples/gke-regional-public-cluster/main.tf @@ -4,17 +4,24 @@ # Load Balancer in front of it. # --------------------------------------------------------------------------------------------------------------------- -provider "google-beta" { - project = "${var.project}" - region = "${var.region}" -} - # Use Terraform 0.10.x so that we can take advantage of Terraform GCP functionality as a separate provider via # https://github.com/terraform-providers/terraform-provider-google terraform { required_version = ">= 0.10.3" } +provider "google" { + version = "~> 2.0.0" + project = "${var.project}" + region = "${var.region}" +} + +provider "google-beta" { + version = "~> 2.0.0" + project = "${var.project}" + region = "${var.region}" +} + module "gke_cluster" { # When using these modules in your own templates, you will need to use a Git URL with a ref attribute that pins you # to a specific version of the modules, such as the following example: