From 0206342763e0c3370cb46278c6ba75dc39436f43 Mon Sep 17 00:00:00 2001 From: Leonardo Henrique Romanini Date: Thu, 2 Jan 2025 15:28:06 -0300 Subject: [PATCH] Add CMEK into Autopilot GKE module --- autogen/main/cluster.tf.tmpl | 5 +- autogen/main/variables.tf.tmpl | 2 - .../simple_autopilot_private_cmek/README.md | 34 ++++++++ .../simple_autopilot_private_cmek/main.tf | 79 +++++++++++++++++++ .../simple_autopilot_private_cmek/network.tf | 50 ++++++++++++ .../simple_autopilot_private_cmek/outputs.tf | 65 +++++++++++++++ .../variables.tf | 19 +++++ .../simple_autopilot_private_cmek/versions.tf | 27 +++++++ .../beta-autopilot-private-cluster/README.md | 1 + .../beta-autopilot-private-cluster/cluster.tf | 5 +- .../variables.tf | 6 ++ .../beta-autopilot-public-cluster/README.md | 1 + .../beta-autopilot-public-cluster/cluster.tf | 5 +- .../variables.tf | 6 ++ .../simple_autopilot_private_cmek_test.go | 46 +++++++++++ 15 files changed, 343 insertions(+), 8 deletions(-) create mode 100644 examples/simple_autopilot_private_cmek/README.md create mode 100644 examples/simple_autopilot_private_cmek/main.tf create mode 100644 examples/simple_autopilot_private_cmek/network.tf create mode 100644 examples/simple_autopilot_private_cmek/outputs.tf create mode 100644 examples/simple_autopilot_private_cmek/variables.tf create mode 100644 examples/simple_autopilot_private_cmek/versions.tf create mode 100644 test/integration/simple_autopilot_private_cmek/simple_autopilot_private_cmek_test.go diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl index d997ac407b..660545da51 100644 --- a/autogen/main/cluster.tf.tmpl +++ b/autogen/main/cluster.tf.tmpl @@ -203,10 +203,11 @@ resource "google_container_cluster" "primary" { {% if autopilot_cluster == true %} cluster_autoscaling { dynamic "auto_provisioning_defaults" { - for_each = (var.create_service_account || var.service_account != "") ? [1] : [] + for_each = (var.create_service_account || var.service_account != "" || var.boot_disk_kms_key != null) ? [1] : [] content { - service_account = local.service_account + service_account = local.service_account + boot_disk_kms_key = var.boot_disk_kms_key } } } diff --git a/autogen/main/variables.tf.tmpl b/autogen/main/variables.tf.tmpl index 704915adf8..73282ec874 100644 --- a/autogen/main/variables.tf.tmpl +++ b/autogen/main/variables.tf.tmpl @@ -447,14 +447,12 @@ variable "service_account_name" { default = "" } -{% if autopilot_cluster != true %} variable "boot_disk_kms_key" { type = string description = "The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY_PROJECT_ID]/locations/[LOCATION]/keyRings/[RING_NAME]/cryptoKeys/[KEY_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption" default = null } -{% endif %} variable "issue_client_certificate" { type = bool description = "Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive!" diff --git a/examples/simple_autopilot_private_cmek/README.md b/examples/simple_autopilot_private_cmek/README.md new file mode 100644 index 0000000000..e6013cf283 --- /dev/null +++ b/examples/simple_autopilot_private_cmek/README.md @@ -0,0 +1,34 @@ +# Simple Regional Autopilot Cluster + +This example illustrates how to create a simple autopilot cluster with beta features and +using a Customer Managed Encryption Keys (CMEK). + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| project\_id | The project ID to host the cluster in | `any` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| boot\_disk\_kms\_key | CMEK used for disk encryption | +| cluster\_name | Cluster name | +| kubernetes\_endpoint | The cluster endpoint | +| location | n/a | +| master\_kubernetes\_version | Kubernetes version of the master | +| network\_name | The name of the VPC being created | +| region | The region in which the cluster resides | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| subnet\_names | The names of the subnet being created | +| zones | List of zones in which the cluster resides | + + + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/examples/simple_autopilot_private_cmek/main.tf b/examples/simple_autopilot_private_cmek/main.tf new file mode 100644 index 0000000000..06d7c5a735 --- /dev/null +++ b/examples/simple_autopilot_private_cmek/main.tf @@ -0,0 +1,79 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cluster_type = "simple-autopilot-private-cmek" + network_name = "simple-autopilot-private-cmek-network" + subnet_name = "simple-autopilot-private-cmek-subnet" + master_auth_subnetwork = "simple-autopilot-private-master-subnet" + pods_range_name = "ip-range-pods-simple-autopilot-private" + svc_range_name = "ip-range-svc-simple-autopilot-private" + subnet_names = [for subnet_self_link in module.gcp-network.subnets_self_links : split("/", subnet_self_link)[length(split("/", subnet_self_link)) - 1]] +} + +data "google_client_config" "default" {} + +data "google_project" "main" { + project_id = var.project_id +} + +module "kms" { + source = "terraform-google-modules/kms/google" + version = "~> 3.2" + + project_id = var.project_id + key_protection_level = "HSM" + location = "us-central1" + keyring = "keyring" + keys = ["key3"] + prevent_destroy = false + # set_owners_for = [ "key3" ] + # owners = ["serviceAccount:service-${data.google_project.main.number}@container-engine-robot.iam.gserviceaccount.com", "serviceAccount:${var.int_sa}", "serviceAccount:service-${data.google_project.main.number}@compute-system.iam.gserviceaccount.com"] +} + +resource "google_kms_crypto_key_iam_member" "main" { + crypto_key_id = values(module.kms.keys)[0] + role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" + member = "serviceAccount:service-${data.google_project.main.number}@compute-system.iam.gserviceaccount.com" +} + +provider "kubernetes" { + host = "https://${module.gke.endpoint}" + token = data.google_client_config.default.access_token + cluster_ca_certificate = base64decode(module.gke.ca_certificate) +} + +module "gke" { + source = "terraform-google-modules/kubernetes-engine/google//modules/beta-autopilot-private-cluster" + version = "~> 35.0" + + project_id = var.project_id + name = "${local.cluster_type}-cluster" + regional = true + region = "us-central1" + network = module.gcp-network.network_name + subnetwork = local.subnet_names[index(module.gcp-network.subnets_names, local.subnet_name)] + ip_range_pods = local.pods_range_name + ip_range_services = local.svc_range_name + release_channel = "REGULAR" + enable_vertical_pod_autoscaling = true + enable_private_endpoint = true + enable_private_nodes = true + network_tags = [local.cluster_type] + deletion_protection = false + boot_disk_kms_key = values(module.kms.keys)[0] + depends_on = [google_kms_crypto_key_iam_member.main] +} diff --git a/examples/simple_autopilot_private_cmek/network.tf b/examples/simple_autopilot_private_cmek/network.tf new file mode 100644 index 0000000000..c5d35d9539 --- /dev/null +++ b/examples/simple_autopilot_private_cmek/network.tf @@ -0,0 +1,50 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "gcp-network" { + source = "terraform-google-modules/network/google" + version = ">= 7.5" + + project_id = var.project_id + network_name = local.network_name + + subnets = [ + { + subnet_name = local.subnet_name + subnet_ip = "10.0.0.0/17" + subnet_region = "us-central1" + subnet_private_access = true + }, + { + subnet_name = local.master_auth_subnetwork + subnet_ip = "10.60.0.0/17" + subnet_region = "us-central1" + }, + ] + + secondary_ranges = { + (local.subnet_name) = [ + { + range_name = local.pods_range_name + ip_cidr_range = "192.168.0.0/18" + }, + { + range_name = local.svc_range_name + ip_cidr_range = "192.168.64.0/18" + }, + ] + } +} diff --git a/examples/simple_autopilot_private_cmek/outputs.tf b/examples/simple_autopilot_private_cmek/outputs.tf new file mode 100644 index 0000000000..c07900485c --- /dev/null +++ b/examples/simple_autopilot_private_cmek/outputs.tf @@ -0,0 +1,65 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "kubernetes_endpoint" { + description = "The cluster endpoint" + sensitive = true + value = module.gke.endpoint +} + +output "cluster_name" { + description = "Cluster name" + value = module.gke.name +} + +output "location" { + value = module.gke.location +} + +output "master_kubernetes_version" { + description = "Kubernetes version of the master" + value = module.gke.master_version +} + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = module.gke.service_account +} + +output "network_name" { + description = "The name of the VPC being created" + value = module.gcp-network.network_name +} + +output "subnet_names" { + description = "The names of the subnet being created" + value = module.gcp-network.subnets_names +} + +output "region" { + description = "The region in which the cluster resides" + value = module.gke.region +} + +output "zones" { + description = "List of zones in which the cluster resides" + value = module.gke.zones +} + +output "boot_disk_kms_key" { + description = "CMEK used for disk encryption" + value = values(module.kms.keys)[0] +} diff --git a/examples/simple_autopilot_private_cmek/variables.tf b/examples/simple_autopilot_private_cmek/variables.tf new file mode 100644 index 0000000000..80f4b3cf3b --- /dev/null +++ b/examples/simple_autopilot_private_cmek/variables.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID to host the cluster in" +} diff --git a/examples/simple_autopilot_private_cmek/versions.tf b/examples/simple_autopilot_private_cmek/versions.tf new file mode 100644 index 0000000000..4c9261fce9 --- /dev/null +++ b/examples/simple_autopilot_private_cmek/versions.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + kubernetes = { + source = "hashicorp/kubernetes" + } + } + required_version = ">= 0.13" +} diff --git a/modules/beta-autopilot-private-cluster/README.md b/modules/beta-autopilot-private-cluster/README.md index 888c450654..2f7c57c46f 100644 --- a/modules/beta-autopilot-private-cluster/README.md +++ b/modules/beta-autopilot-private-cluster/README.md @@ -77,6 +77,7 @@ Then perform the following commands on the root folder: | additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | | allow\_net\_admin | (Optional) Enable NET\_ADMIN for the cluster. | `bool` | `null` | no | | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | +| boot\_disk\_kms\_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY\_PROJECT\_ID]/locations/[LOCATION]/keyRings/[RING\_NAME]/cryptoKeys/[KEY\_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption | `string` | `null` | no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `string` | `null` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | diff --git a/modules/beta-autopilot-private-cluster/cluster.tf b/modules/beta-autopilot-private-cluster/cluster.tf index bddef25b74..16e17ae693 100644 --- a/modules/beta-autopilot-private-cluster/cluster.tf +++ b/modules/beta-autopilot-private-cluster/cluster.tf @@ -89,10 +89,11 @@ resource "google_container_cluster" "primary" { cluster_autoscaling { dynamic "auto_provisioning_defaults" { - for_each = (var.create_service_account || var.service_account != "") ? [1] : [] + for_each = (var.create_service_account || var.service_account != "" || var.boot_disk_kms_key != null) ? [1] : [] content { - service_account = local.service_account + service_account = local.service_account + boot_disk_kms_key = var.boot_disk_kms_key } } } diff --git a/modules/beta-autopilot-private-cluster/variables.tf b/modules/beta-autopilot-private-cluster/variables.tf index 813d42b716..ef6a20a3f1 100644 --- a/modules/beta-autopilot-private-cluster/variables.tf +++ b/modules/beta-autopilot-private-cluster/variables.tf @@ -240,6 +240,12 @@ variable "service_account_name" { default = "" } +variable "boot_disk_kms_key" { + type = string + description = "The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY_PROJECT_ID]/locations/[LOCATION]/keyRings/[RING_NAME]/cryptoKeys/[KEY_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption" + default = null +} + variable "issue_client_certificate" { type = bool description = "Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive!" diff --git a/modules/beta-autopilot-public-cluster/README.md b/modules/beta-autopilot-public-cluster/README.md index 729039e281..baaea7e4c8 100644 --- a/modules/beta-autopilot-public-cluster/README.md +++ b/modules/beta-autopilot-public-cluster/README.md @@ -72,6 +72,7 @@ Then perform the following commands on the root folder: | additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | | allow\_net\_admin | (Optional) Enable NET\_ADMIN for the cluster. | `bool` | `null` | no | | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | +| boot\_disk\_kms\_key | The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY\_PROJECT\_ID]/locations/[LOCATION]/keyRings/[RING\_NAME]/cryptoKeys/[KEY\_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption | `string` | `null` | no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | `string` | `null` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | `map(string)` | `{}` | no | | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | `bool` | `false` | no | diff --git a/modules/beta-autopilot-public-cluster/cluster.tf b/modules/beta-autopilot-public-cluster/cluster.tf index 4baca3ecac..2cfc931c13 100644 --- a/modules/beta-autopilot-public-cluster/cluster.tf +++ b/modules/beta-autopilot-public-cluster/cluster.tf @@ -89,10 +89,11 @@ resource "google_container_cluster" "primary" { cluster_autoscaling { dynamic "auto_provisioning_defaults" { - for_each = (var.create_service_account || var.service_account != "") ? [1] : [] + for_each = (var.create_service_account || var.service_account != "" || var.boot_disk_kms_key != null) ? [1] : [] content { - service_account = local.service_account + service_account = local.service_account + boot_disk_kms_key = var.boot_disk_kms_key } } } diff --git a/modules/beta-autopilot-public-cluster/variables.tf b/modules/beta-autopilot-public-cluster/variables.tf index f212deff44..056cf4f1ba 100644 --- a/modules/beta-autopilot-public-cluster/variables.tf +++ b/modules/beta-autopilot-public-cluster/variables.tf @@ -240,6 +240,12 @@ variable "service_account_name" { default = "" } +variable "boot_disk_kms_key" { + type = string + description = "The Customer Managed Encryption Key used to encrypt the boot disk attached to each node in the node pool, if not overridden in `node_pools`. This should be of the form projects/[KEY_PROJECT_ID]/locations/[LOCATION]/keyRings/[RING_NAME]/cryptoKeys/[KEY_NAME]. For more information about protecting resources with Cloud KMS Keys please see: https://cloud.google.com/compute/docs/disks/customer-managed-encryption" + default = null +} + variable "issue_client_certificate" { type = bool description = "Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive!" diff --git a/test/integration/simple_autopilot_private_cmek/simple_autopilot_private_cmek_test.go b/test/integration/simple_autopilot_private_cmek/simple_autopilot_private_cmek_test.go new file mode 100644 index 0000000000..1667ae79df --- /dev/null +++ b/test/integration/simple_autopilot_private_cmek/simple_autopilot_private_cmek_test.go @@ -0,0 +1,46 @@ +// Copyright 2022-2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package simple_autopilot_private_cmek + +import ( + "testing" + "time" + + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" + "github.com/stretchr/testify/assert" + "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/testutils" +) + +func TestSimpleAutopilotPrivateCMEK(t *testing.T) { + projectID := testutils.GetTestProjectFromSetup(t, 1) + bpt := tft.NewTFBlueprintTest(t, + tft.WithVars(map[string]interface{}{"project_id": projectID}), + tft.WithRetryableTerraformErrors(testutils.RetryableTransientErrors, 3, 2*time.Minute), + ) + + bpt.DefineVerify(func(assert *assert.Assertions) { + bpt.DefaultVerify(assert) + + location := bpt.GetStringOutput("location") + clusterName := bpt.GetStringOutput("cluster_name") + key := bpt.GetStringOutput("boot_disk_kms_key") + + op := gcloud.Runf(t, "container clusters describe %s --zone %s --project %s", clusterName, location, projectID) + assert.True(op.Get("autopilot.enabled").Bool(), "should be autopilot") + assert.Equal(op.Get("autoscaling.autoprovisioningNodePoolDefaults.bootDiskKmsKey").String(), key, "should have CMEK configured") + }) + bpt.Test() +}