diff --git a/_examples/aks/aks-cluster/main.tf b/_examples/aks/aks-cluster/main.tf index 1a0648910b..f6931069ff 100644 --- a/_examples/aks/aks-cluster/main.tf +++ b/_examples/aks/aks-cluster/main.tf @@ -1,16 +1,16 @@ -resource "azurerm_resource_group" "test" { +resource "azurerm_resource_group" "default" { name = var.cluster_name location = var.location } -resource "azurerm_kubernetes_cluster" "test" { +resource "azurerm_kubernetes_cluster" "default" { name = var.cluster_name - location = azurerm_resource_group.test.location - resource_group_name = azurerm_resource_group.test.name + location = azurerm_resource_group.default.location + resource_group_name = azurerm_resource_group.default.name dns_prefix = var.cluster_name default_node_pool { - name = "default" + name = "test" node_count = 1 vm_size = "Standard_DS2_v2" } @@ -19,9 +19,3 @@ resource "azurerm_kubernetes_cluster" "test" { type = "SystemAssigned" } } - -resource "local_file" "kubeconfig" { - content = azurerm_kubernetes_cluster.test.kube_config_raw - filename = "${path.root}/kubeconfig" -} - diff --git a/_examples/aks/aks-cluster/output.tf b/_examples/aks/aks-cluster/output.tf deleted file mode 100644 index 9bb8518983..0000000000 --- a/_examples/aks/aks-cluster/output.tf +++ /dev/null @@ -1,15 +0,0 @@ -output "client_cert" { - value = azurerm_kubernetes_cluster.test.kube_config.0.client_certificate -} - -output "client_key" { - value = azurerm_kubernetes_cluster.test.kube_config.0.client_key -} - -output "ca_cert" { - value = azurerm_kubernetes_cluster.test.kube_config.0.cluster_ca_certificate -} - -output "endpoint" { - value = azurerm_kubernetes_cluster.test.kube_config.0.host -} diff --git a/_examples/aks/kubernetes-config/main.tf b/_examples/aks/kubernetes-config/main.tf index 3eeabc804d..5cc51b2103 100644 --- a/_examples/aks/kubernetes-config/main.tf +++ b/_examples/aks/kubernetes-config/main.tf @@ -54,3 +54,8 @@ resource helm_release nginx_ingress { value = "ClusterIP" } } + +resource "local_file" "kubeconfig" { + content = var.kubeconfig + filename = "${path.root}/kubeconfig" +} diff --git a/_examples/aks/kubernetes-config/variables.tf b/_examples/aks/kubernetes-config/variables.tf index abbf86f798..e59bc9bf25 100644 --- a/_examples/aks/kubernetes-config/variables.tf +++ b/_examples/aks/kubernetes-config/variables.tf @@ -1,3 +1,7 @@ variable "cluster_name" { type = string } + +variable "kubeconfig" { + type = string +} diff --git a/_examples/aks/main.tf b/_examples/aks/main.tf index 91e08dfaac..46d7f4047a 100644 --- a/_examples/aks/main.tf +++ b/_examples/aks/main.tf @@ -2,7 +2,7 @@ terraform { required_providers { kubernetes = { source = "hashicorp/kubernetes" - version = ">= 2.0.0" + version = ">= 2.0.2" } azurerm = { source = "hashicorp/azurerm" @@ -10,24 +10,30 @@ terraform { } helm = { source = "hashicorp/helm" - version = ">= 2.0.1" + version = ">= 2.0.2" } } } +data "azurerm_kubernetes_cluster" "default" { + depends_on = [module.aks-cluster] # refresh cluster state before reading + name = local.cluster_name + resource_group_name = local.cluster_name +} + provider "kubernetes" { - host = module.aks-cluster.endpoint - client_key = base64decode(module.aks-cluster.client_key) - client_certificate = base64decode(module.aks-cluster.client_cert) - cluster_ca_certificate = base64decode(module.aks-cluster.ca_cert) + host = data.azurerm_kubernetes_cluster.default.kube_config.0.host + client_certificate = base64decode(data.azurerm_kubernetes_cluster.default.kube_config.0.client_certificate) + client_key = base64decode(data.azurerm_kubernetes_cluster.default.kube_config.0.client_key) + cluster_ca_certificate = base64decode(data.azurerm_kubernetes_cluster.default.kube_config.0.cluster_ca_certificate) } provider "helm" { kubernetes { - host = module.aks-cluster.endpoint - client_key = base64decode(module.aks-cluster.client_key) - client_certificate = base64decode(module.aks-cluster.client_cert) - cluster_ca_certificate = base64decode(module.aks-cluster.ca_cert) + host = data.azurerm_kubernetes_cluster.default.kube_config.0.host + client_certificate = base64decode(data.azurerm_kubernetes_cluster.default.kube_config.0.client_certificate) + client_key = base64decode(data.azurerm_kubernetes_cluster.default.kube_config.0.client_key) + cluster_ca_certificate = base64decode(data.azurerm_kubernetes_cluster.default.kube_config.0.cluster_ca_certificate) } } @@ -36,15 +42,14 @@ provider "azurerm" { } module "aks-cluster" { - providers = { azurerm = azurerm } source = "./aks-cluster" cluster_name = local.cluster_name location = var.location } module "kubernetes-config" { - providers = { kubernetes = kubernetes, helm = helm } depends_on = [module.aks-cluster] source = "./kubernetes-config" cluster_name = local.cluster_name + kubeconfig = data.azurerm_kubernetes_cluster.default.kube_config_raw } diff --git a/_examples/eks/kubernetes-config/main.tf b/_examples/eks/kubernetes-config/main.tf index 2c963afcbd..c1417fcdc9 100644 --- a/_examples/eks/kubernetes-config/main.tf +++ b/_examples/eks/kubernetes-config/main.tf @@ -1,15 +1,4 @@ -provider "kubernetes" { - host = var.cluster_endpoint - cluster_ca_certificate = base64decode(var.cluster_ca_cert) - exec { - api_version = "client.authentication.k8s.io/v1alpha1" - args = ["eks", "get-token", "--cluster-name", var.cluster_name] - command = "aws" - } -} - resource "kubernetes_config_map" "name" { - depends_on = [var.cluster_name] metadata { name = "aws-auth" namespace = "kube-system" @@ -23,26 +12,20 @@ resource "kubernetes_config_map" "name" { } } -# This allows the kubeconfig file to be refreshed during every Terraform apply. # Optional: this kubeconfig file is only used for manual CLI access to the cluster. resource "null_resource" "generate-kubeconfig" { provisioner "local-exec" { command = "aws eks update-kubeconfig --name ${var.cluster_name} --kubeconfig ${path.root}/kubeconfig" } - triggers = { - always_run = timestamp() - } } resource "kubernetes_namespace" "test" { - depends_on = [var.cluster_name] metadata { name = "test" } } resource "kubernetes_deployment" "test" { - depends_on = [var.cluster_name] metadata { name = "test" namespace= kubernetes_namespace.test.metadata.0.name @@ -81,20 +64,7 @@ resource "kubernetes_deployment" "test" { } } -provider "helm" { - kubernetes { - host = var.cluster_endpoint - cluster_ca_certificate = base64decode(var.cluster_ca_cert) - exec { - api_version = "client.authentication.k8s.io/v1alpha1" - args = ["eks", "get-token", "--cluster-name", var.cluster_name] - command = "aws" - } - } -} - resource helm_release nginx_ingress { - depends_on = [var.cluster_name] name = "nginx-ingress-controller" repository = "https://charts.bitnami.com/bitnami" diff --git a/_examples/eks/kubernetes-config/variables.tf b/_examples/eks/kubernetes-config/variables.tf index 169c564f82..af19cf115c 100644 --- a/_examples/eks/kubernetes-config/variables.tf +++ b/_examples/eks/kubernetes-config/variables.tf @@ -1,12 +1,4 @@ variable "k8s_node_role_arn" { - type = list(string) -} - -variable "cluster_ca_cert" { - type = string -} - -variable "cluster_endpoint" { type = string } diff --git a/_examples/eks/main.tf b/_examples/eks/main.tf index 927b7f1457..a44bf596c0 100644 --- a/_examples/eks/main.tf +++ b/_examples/eks/main.tf @@ -15,6 +15,34 @@ terraform { } } +data "aws_eks_cluster" "default" { + name = module.cluster.cluster_id +} + +data "aws_eks_cluster_auth" "default" { + name = module.cluster.cluster_id +} + +provider "kubernetes" { + host = data.aws_eks_cluster.default.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.default.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.default.token +} + +provider "helm" { + kubernetes { + host = data.aws_eks_cluster.default.endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.default.certificate_authority[0].data) + token = data.aws_eks_cluster_auth.default.token + } +} + +# exec { +# api_version = "client.authentication.k8s.io/v1alpha1" +# args = ["eks", "get-token", "--cluster-name", var.cluster_name] +# command = "aws" +# } + provider "aws" { region = var.region } @@ -51,9 +79,7 @@ module "cluster" { } module "kubernetes-config" { - source = "./kubernetes-config" - k8s_node_role_arn = list(module.cluster.worker_iam_role_arn) - cluster_ca_cert = module.cluster.cluster_certificate_authority_data cluster_name = module.cluster.cluster_id # creates dependency on cluster creation - cluster_endpoint = module.cluster.cluster_endpoint + source = "./kubernetes-config" + k8s_node_role_arn = module.cluster.worker_iam_role_arn } diff --git a/_examples/gke/gke-cluster/main.tf b/_examples/gke/gke-cluster/main.tf index 87a8f88123..171804ef52 100644 --- a/_examples/gke/gke-cluster/main.tf +++ b/_examples/gke/gke-cluster/main.tf @@ -1,9 +1,3 @@ -provider "google" { - # Provider is configured using environment variables: GOOGLE_REGION, GOOGLE_PROJECT, GOOGLE_CREDENTIALS. - # This can be set statically, if preferred. See docs for details. - # https://registry.terraform.io/providers/hashicorp/google/latest/docs/guides/provider_reference#full-reference -} - # This is used to set local variable google_zone. # This can be replaced with a statically-configured zone, if preferred. data "google_compute_zones" "available" { @@ -14,7 +8,7 @@ data "google_container_engine_versions" "supported" { version_prefix = var.kubernetes_version } -resource "google_container_cluster" "primary" { +resource "google_container_cluster" "default" { name = var.cluster_name location = local.google_zone initial_node_count = var.workers_count diff --git a/_examples/gke/gke-cluster/output.tf b/_examples/gke/gke-cluster/output.tf index 500403d0dc..df57a2f024 100644 --- a/_examples/gke/gke-cluster/output.tf +++ b/_examples/gke/gke-cluster/output.tf @@ -1,21 +1,5 @@ output "node_version" { - value = google_container_cluster.primary.node_version -} - -output "cluster_id" { - value = google_container_cluster.primary.id -} - -output "cluster_endpoint" { - value = google_container_cluster.primary.endpoint -} - -output "cluster_ca_cert" { - value = google_container_cluster.primary.master_auth[0].cluster_ca_certificate -} - -output "cluster_name" { - value = google_container_cluster.primary.name + value = google_container_cluster.default.node_version } output "google_zone" { diff --git a/_examples/gke/kubernetes-config/kubeconfig-template.yaml b/_examples/gke/kubeconfig-template.yaml similarity index 91% rename from _examples/gke/kubernetes-config/kubeconfig-template.yaml rename to _examples/gke/kubeconfig-template.yaml index 221999be7b..d5d1b6e2a2 100644 --- a/_examples/gke/kubernetes-config/kubeconfig-template.yaml +++ b/_examples/gke/kubeconfig-template.yaml @@ -2,7 +2,7 @@ apiVersion: v1 clusters: - cluster: certificate-authority-data: ${cluster_ca} - server: https://${endpoint} + server: ${endpoint} name: ${cluster_name} contexts: - context: diff --git a/_examples/gke/kubernetes-config/main.tf b/_examples/gke/kubernetes-config/main.tf index ac3ef4effc..3eeabc804d 100644 --- a/_examples/gke/kubernetes-config/main.tf +++ b/_examples/gke/kubernetes-config/main.tf @@ -1,15 +1,3 @@ -# Configure kubernetes provider with Oauth2 access token. -# https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config -# This fetches a new token, which will expire in 1 hour. -data "google_client_config" "default" { -} - -provider "kubernetes" { - host = var.cluster_endpoint - token = data.google_client_config.default.access_token - cluster_ca_certificate = base64decode(var.cluster_ca_cert) -} - resource "kubernetes_namespace" "test" { metadata { name = "test" @@ -55,14 +43,6 @@ resource "kubernetes_deployment" "test" { } } -provider "helm" { - kubernetes { - host = var.cluster_endpoint - token = data.google_client_config.default.access_token - cluster_ca_certificate = base64decode(var.cluster_ca_cert) - } -} - resource helm_release nginx_ingress { name = "nginx-ingress-controller" @@ -74,21 +54,3 @@ resource helm_release nginx_ingress { value = "ClusterIP" } } - -data "template_file" "kubeconfig" { - template = file("${path.module}/kubeconfig-template.yaml") - - vars = { - cluster_name = var.cluster_name - endpoint = var.cluster_endpoint - cluster_ca = var.cluster_ca_cert - cluster_token = data.google_client_config.default.access_token - } -} - -resource "local_file" "kubeconfig" { - depends_on = [var.cluster_id] - content = data.template_file.kubeconfig.rendered - filename = "${path.root}/kubeconfig" -} - diff --git a/_examples/gke/kubernetes-config/variables.tf b/_examples/gke/kubernetes-config/variables.tf index 8286692e5e..abbf86f798 100644 --- a/_examples/gke/kubernetes-config/variables.tf +++ b/_examples/gke/kubernetes-config/variables.tf @@ -1,16 +1,3 @@ variable "cluster_name" { type = string } - -variable "cluster_id" { - type = string -} - -variable "cluster_endpoint" { - type = string -} - -variable "cluster_ca_cert" { - type = string -} - diff --git a/_examples/gke/main.tf b/_examples/gke/main.tf index 34647aa911..7c089e324c 100644 --- a/_examples/gke/main.tf +++ b/_examples/gke/main.tf @@ -2,7 +2,7 @@ terraform { required_providers { kubernetes = { source = "hashicorp/kubernetes" - version = ">= 2.0.0" + version = ">= 9.9.9" } google = { source = "hashicorp/google" @@ -15,12 +15,40 @@ terraform { } } -resource "random_id" "cluster_name" { - byte_length = 5 +# Provider is configured using environment variables: GOOGLE_REGION, GOOGLE_PROJECT, GOOGLE_CREDENTIALS. +# This can be set statically, if preferred. See docs for details. +# https://registry.terraform.io/providers/hashicorp/google/latest/docs/guides/provider_reference#full-reference +provider "google" {} + +# Configure kubernetes provider with Oauth2 access token. +# https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config +# This fetches a new token, which will expire in 1 hour. +data "google_client_config" "default" { + depends_on = [module.gke-cluster] +} + +# Defer reading the cluster data until the latest ca_cert data exists. +data "google_container_cluster" "default" { + name = local.cluster_name + depends_on = [module.gke-cluster] } -locals { - cluster_name = "tf-k8s-${random_id.cluster_name.hex}" +provider "kubernetes" { + host = "https://${data.google_container_cluster.default.endpoint}" + token = data.google_client_config.default.access_token + cluster_ca_certificate = base64decode( + data.google_container_cluster.default.master_auth[0].cluster_ca_certificate, + ) +} + +provider "helm" { + kubernetes { + host = "https://${data.google_container_cluster.default.endpoint}" + token = data.google_client_config.default.access_token + cluster_ca_certificate = base64decode( + data.google_container_cluster.default.master_auth[0].cluster_ca_certificate, + ) + } } module "gke-cluster" { @@ -29,10 +57,26 @@ module "gke-cluster" { } module "kubernetes-config" { + depends_on = [module.gke-cluster] source = "./kubernetes-config" - cluster_name = module.gke-cluster.cluster_name - cluster_id = module.gke-cluster.cluster_id # creates dependency on cluster creation - cluster_endpoint = module.gke-cluster.cluster_endpoint - cluster_ca_cert = module.gke-cluster.cluster_ca_cert + cluster_name = local.cluster_name +} + +# optional: used for manual CLI access to the cluster when gcloud tool is unavailable. +# The gcloud tool can make a longer-lived kubeconfig. This one expires in one hour and can be updated using `terraform apply`. +data "template_file" "kubeconfig" { + template = file("kubeconfig-template.yaml") + + vars = { + cluster_name = local.cluster_name + endpoint = "https://${data.google_container_cluster.default.endpoint}" + cluster_ca = data.google_container_cluster.default.master_auth[0].cluster_ca_certificate + cluster_token = data.google_client_config.default.access_token + } +} + +resource "local_file" "kubeconfig" { + content = data.template_file.kubeconfig.rendered + filename = "${path.root}/kubeconfig" } diff --git a/_examples/gke/variables.tf b/_examples/gke/variables.tf new file mode 100644 index 0000000000..4c9484ca52 --- /dev/null +++ b/_examples/gke/variables.tf @@ -0,0 +1,7 @@ +resource "random_id" "cluster_name" { + byte_length = 5 +} + +locals { + cluster_name = "tf-k8s-${random_id.cluster_name.hex}" +} diff --git a/kubernetes/test-infra/eks/main.tf b/kubernetes/test-infra/eks/main.tf index 3058d02e40..b6aa84cea5 100644 --- a/kubernetes/test-infra/eks/main.tf +++ b/kubernetes/test-infra/eks/main.tf @@ -47,7 +47,7 @@ module "cluster" { module "node-config" { source = "./node-config" - k8s_node_role_arn = list(module.cluster.worker_iam_role_arn) + k8s_node_role_arn = tolist(module.cluster.worker_iam_role_arn) cluster_ca = module.cluster.cluster_certificate_authority_data cluster_name = module.cluster.cluster_id # creates dependency on cluster creation cluster_endpoint = module.cluster.cluster_endpoint diff --git a/kubernetes/test-infra/eks/node-config/variables.tf b/kubernetes/test-infra/eks/node-config/variables.tf index a1868fde92..aed4df62ff 100644 --- a/kubernetes/test-infra/eks/node-config/variables.tf +++ b/kubernetes/test-infra/eks/node-config/variables.tf @@ -1,5 +1,5 @@ variable "k8s_node_role_arn" { - type = list(string) + type = tolist(string) } variable "cluster_ca" {