Add CMEK into Autopilot GKE module

terraform-google-modules · Jan 2, 2025 · 0206342 · 0206342
1 parent 49478f5
commit 0206342
Show file tree

Hide file tree

Showing 15 changed files with 343 additions and 8 deletions.
diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl
@@ -203,10 +203,11 @@ resource "google_container_cluster" "primary" {
   {% if autopilot_cluster == true %}
   cluster_autoscaling {
     dynamic "auto_provisioning_defaults" {
-      for_each = (var.create_service_account || var.service_account != "") ? [1] : []
+      for_each = (var.create_service_account || var.service_account != "" || var.boot_disk_kms_key != null) ? [1] : []
 
       content {
-        service_account = local.service_account
+        service_account   = local.service_account
+        boot_disk_kms_key = var.boot_disk_kms_key
       }
     }
   }

diff --git a/autogen/main/variables.tf.tmpl b/autogen/main/variables.tf.tmpl
@@ -447,14 +447,12 @@ variable "service_account_name" {
   default     = ""
 }
 
-{% if autopilot_cluster != true %}
 variable "boot_disk_kms_key" {
   type        = string
   description = "The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY_PROJECT_ID]/locations/[LOCATION]/keyRings/[RING_NAME]/cryptoKeys/[KEY_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption"
   default     = null
 }
 
-{% endif %}
 variable "issue_client_certificate" {
   type        = bool
   description = "Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive!"

diff --git a/examples/simple_autopilot_private_cmek/README.md b/examples/simple_autopilot_private_cmek/README.md
@@ -0,0 +1,34 @@
+# Simple Regional Autopilot Cluster
+
+This example illustrates how to create a simple autopilot cluster with beta features and
+using a Customer Managed Encryption Keys (CMEK).
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| project\_id | The project ID to host the cluster in | `any` | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| boot\_disk\_kms\_key | CMEK used for disk encryption |
+| cluster\_name | Cluster name |
+| kubernetes\_endpoint | The cluster endpoint |
+| location | n/a |
+| master\_kubernetes\_version | Kubernetes version of the master |
+| network\_name | The name of the VPC being created |
+| region | The region in which the cluster resides |
+| service\_account | The service account to default running nodes as if not overridden in `node_pools`. |
+| subnet\_names | The names of the subnet being created |
+| zones | List of zones in which the cluster resides |
+
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+
+To provision this example, run the following from within this directory:
+- `terraform init` to get the plugins
+- `terraform plan` to see the infrastructure plan
+- `terraform apply` to apply the infrastructure build
+- `terraform destroy` to destroy the built infrastructure
diff --git a/examples/simple_autopilot_private_cmek/main.tf b/examples/simple_autopilot_private_cmek/main.tf
@@ -0,0 +1,79 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+locals {
+  cluster_type           = "simple-autopilot-private-cmek"
+  network_name           = "simple-autopilot-private-cmek-network"
+  subnet_name            = "simple-autopilot-private-cmek-subnet"
+  master_auth_subnetwork = "simple-autopilot-private-master-subnet"
+  pods_range_name        = "ip-range-pods-simple-autopilot-private"
+  svc_range_name         = "ip-range-svc-simple-autopilot-private"
+  subnet_names           = [for subnet_self_link in module.gcp-network.subnets_self_links : split("/", subnet_self_link)[length(split("/", subnet_self_link)) - 1]]
+}
+
+data "google_client_config" "default" {}
+
+data "google_project" "main" {
+  project_id = var.project_id
+}
+
+module "kms" {
+  source  = "terraform-google-modules/kms/google"
+  version = "~> 3.2"
+
+  project_id           = var.project_id
+  key_protection_level = "HSM"
+  location             = "us-central1"
+  keyring              = "keyring"
+  keys                 = ["key3"]
+  prevent_destroy      = false
+  # set_owners_for = [ "key3" ]
+  # owners = ["serviceAccount:service-${data.google_project.main.number}@container-engine-robot.iam.gserviceaccount.com", "serviceAccount:${var.int_sa}", "serviceAccount:service-${data.google_project.main.number}@compute-system.iam.gserviceaccount.com"]
+}
+
+resource "google_kms_crypto_key_iam_member" "main" {
+  crypto_key_id = values(module.kms.keys)[0]
+  role          = "roles/cloudkms.cryptoKeyEncrypterDecrypter"
+  member        = "serviceAccount:service-${data.google_project.main.number}@compute-system.iam.gserviceaccount.com"
+}
+
+provider "kubernetes" {
+  host                   = "https://${module.gke.endpoint}"
+  token                  = data.google_client_config.default.access_token
+  cluster_ca_certificate = base64decode(module.gke.ca_certificate)
+}
+
+module "gke" {
+  source  = "terraform-google-modules/kubernetes-engine/google//modules/beta-autopilot-private-cluster"
+  version = "~> 35.0"
+
+  project_id                      = var.project_id
+  name                            = "${local.cluster_type}-cluster"
+  regional                        = true
+  region                          = "us-central1"
+  network                         = module.gcp-network.network_name
+  subnetwork                      = local.subnet_names[index(module.gcp-network.subnets_names, local.subnet_name)]
+  ip_range_pods                   = local.pods_range_name
+  ip_range_services               = local.svc_range_name
+  release_channel                 = "REGULAR"
+  enable_vertical_pod_autoscaling = true
+  enable_private_endpoint         = true
+  enable_private_nodes            = true
+  network_tags                    = [local.cluster_type]
+  deletion_protection             = false
+  boot_disk_kms_key               = values(module.kms.keys)[0]
+  depends_on                      = [google_kms_crypto_key_iam_member.main]
+}
diff --git a/examples/simple_autopilot_private_cmek/network.tf b/examples/simple_autopilot_private_cmek/network.tf
@@ -0,0 +1,50 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module "gcp-network" {
+  source  = "terraform-google-modules/network/google"
+  version = ">= 7.5"
+
+  project_id   = var.project_id
+  network_name = local.network_name
+
+  subnets = [
+    {
+      subnet_name           = local.subnet_name
+      subnet_ip             = "10.0.0.0/17"
+      subnet_region         = "us-central1"
+      subnet_private_access = true
+    },
+    {
+      subnet_name   = local.master_auth_subnetwork
+      subnet_ip     = "10.60.0.0/17"
+      subnet_region = "us-central1"
+    },
+  ]
+
+  secondary_ranges = {
+    (local.subnet_name) = [
+      {
+        range_name    = local.pods_range_name
+        ip_cidr_range = "192.168.0.0/18"
+      },
+      {
+        range_name    = local.svc_range_name
+        ip_cidr_range = "192.168.64.0/18"
+      },
+    ]
+  }
+}
diff --git a/examples/simple_autopilot_private_cmek/outputs.tf b/examples/simple_autopilot_private_cmek/outputs.tf
@@ -0,0 +1,65 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output "kubernetes_endpoint" {
+  description = "The cluster endpoint"
+  sensitive   = true
+  value       = module.gke.endpoint
+}
+
+output "cluster_name" {
+  description = "Cluster name"
+  value       = module.gke.name
+}
+
+output "location" {
+  value = module.gke.location
+}
+
+output "master_kubernetes_version" {
+  description = "Kubernetes version of the master"
+  value       = module.gke.master_version
+}
+
+output "service_account" {
+  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  value       = module.gke.service_account
+}
+
+output "network_name" {
+  description = "The name of the VPC being created"
+  value       = module.gcp-network.network_name
+}
+
+output "subnet_names" {
+  description = "The names of the subnet being created"
+  value       = module.gcp-network.subnets_names
+}
+
+output "region" {
+  description = "The region in which the cluster resides"
+  value       = module.gke.region
+}
+
+output "zones" {
+  description = "List of zones in which the cluster resides"
+  value       = module.gke.zones
+}
+
+output "boot_disk_kms_key" {
+  description = "CMEK used for disk encryption"
+  value       = values(module.kms.keys)[0]
+}
diff --git a/examples/simple_autopilot_private_cmek/variables.tf b/examples/simple_autopilot_private_cmek/variables.tf
@@ -0,0 +1,19 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "project_id" {
+  description = "The project ID to host the cluster in"
+}
diff --git a/examples/simple_autopilot_private_cmek/versions.tf b/examples/simple_autopilot_private_cmek/versions.tf
@@ -0,0 +1,27 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+terraform {
+  required_providers {
+    google = {
+      source = "hashicorp/google"
+    }
+    kubernetes = {
+      source = "hashicorp/kubernetes"
+    }
+  }
+  required_version = ">= 0.13"
+}
diff --git a/modules/beta-autopilot-private-cluster/README.md b/modules/beta-autopilot-private-cluster/README.md
@@ -77,6 +77,7 @@ Then perform the following commands on the root folder:
 | additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no |
 | allow\_net\_admin | (Optional) Enable NET\_ADMIN for the cluster. | `bool` | `null` | no |
 | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format [email protected] | `string` | `null` | no |
+| boot\_disk\_kms\_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY\_PROJECT\_ID]/locations/[LOCATION]/keyRings/[RING\_NAME]/cryptoKeys/[KEY\_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption | `string` | `null` | no |
 | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `string` | `null` | no |
 | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no |
 | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no |

diff --git a/modules/beta-autopilot-private-cluster/cluster.tf b/modules/beta-autopilot-private-cluster/cluster.tf
@@ -89,10 +89,11 @@ resource "google_container_cluster" "primary" {
 
   cluster_autoscaling {
     dynamic "auto_provisioning_defaults" {
-      for_each = (var.create_service_account || var.service_account != "") ? [1] : []
+      for_each = (var.create_service_account || var.service_account != "" || var.boot_disk_kms_key != null) ? [1] : []
 
       content {
-        service_account = local.service_account
+        service_account   = local.service_account
+        boot_disk_kms_key = var.boot_disk_kms_key
       }
     }
   }

diff --git a/modules/beta-autopilot-private-cluster/variables.tf b/modules/beta-autopilot-private-cluster/variables.tf
@@ -240,6 +240,12 @@ variable "service_account_name" {
   default     = ""
 }
 
+variable "boot_disk_kms_key" {
+  type        = string
+  description = "The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY_PROJECT_ID]/locations/[LOCATION]/keyRings/[RING_NAME]/cryptoKeys/[KEY_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption"
+  default     = null
+}
+
 variable "issue_client_certificate" {
   type        = bool
   description = "Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive!"

diff --git a/modules/beta-autopilot-public-cluster/README.md b/modules/beta-autopilot-public-cluster/README.md
@@ -72,6 +72,7 @@ Then perform the following commands on the root folder:
 | additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no |
 | allow\_net\_admin | (Optional) Enable NET\_ADMIN for the cluster. | `bool` | `null` | no |
 | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format [email protected] | `string` | `null` | no |
+| boot\_disk\_kms\_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY\_PROJECT\_ID]/locations/[LOCATION]/keyRings/[RING\_NAME]/cryptoKeys/[KEY\_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption | `string` | `null` | no |
 | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `string` | `null` | no |
 | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no |
 | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no |

diff --git a/modules/beta-autopilot-public-cluster/cluster.tf b/modules/beta-autopilot-public-cluster/cluster.tf
@@ -89,10 +89,11 @@ resource "google_container_cluster" "primary" {
 
   cluster_autoscaling {
     dynamic "auto_provisioning_defaults" {
-      for_each = (var.create_service_account || var.service_account != "") ? [1] : []
+      for_each = (var.create_service_account || var.service_account != "" || var.boot_disk_kms_key != null) ? [1] : []
 
       content {
-        service_account = local.service_account
+        service_account   = local.service_account
+        boot_disk_kms_key = var.boot_disk_kms_key
       }
     }
   }

diff --git a/modules/beta-autopilot-public-cluster/variables.tf b/modules/beta-autopilot-public-cluster/variables.tf
@@ -240,6 +240,12 @@ variable "service_account_name" {
   default     = ""
 }
 
+variable "boot_disk_kms_key" {
+  type        = string
+  description = "The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY_PROJECT_ID]/locations/[LOCATION]/keyRings/[RING_NAME]/cryptoKeys/[KEY_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption"
+  default     = null
+}
+
 variable "issue_client_certificate" {
   type        = bool
   description = "Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive!"