Fixes #180: Add tests for beta submodules/examples

.kitchen.yml

@@ -162,6 +162,19 @@ suites:
       systems:
         - name: workload_metadata_config
           backend: local
+  - name: "beta_cluster"
+    driver:
+      root_module_directory: test/fixtures/beta_cluster
+    verifier:
+      systems:
+        - name: gcloud
+          backend: local
+          controls:
+            - gcloud
+        - name: gcp
+          backend: gcp
+          controls:
+            - gcp
   - name: "deploy_service"
     driver:
       root_module_directory: test/fixtures/deploy_service

build/int.cloudbuild.yaml

@@ -264,6 +264,26 @@ steps:
     - verify workload-metadata-config-local
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy workload-metadata-config-local']
+- id: create beta-cluster-local
+  waitFor:
+    - prepare
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create beta-cluster-local']
+- id: converge beta-cluster-local
+  waitFor:
+    - create beta-cluster-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge beta-cluster-local']
+- id: verify beta-cluster-local
+  waitFor:
+    - converge beta-cluster-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify beta-cluster-local']
+#- id: destroy beta-cluster-local
+#  waitFor:
+#    - verify beta-cluster-local
+#  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+#  args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy beta-cluster-local']
 - id: create deploy-service-local
   waitFor:
     - prepare

examples/simple_regional_beta/README.md

@@ -10,17 +10,22 @@ This example illustrates how to create a simple cluster with beta features.
 | cloudrun | Boolean to enable / disable CloudRun | string | `"true"` | no |
 | cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no |
 | compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes |
+| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `<list>` | no |
+| enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no |
 | ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes |
 | ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes |
 | istio | Boolean to enable / disable Istio | string | `"true"` | no |
 | network | The VPC network to host the cluster in | string | n/a | yes |
 | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no |
 | node\_pools | List of maps containing node pools | list(map(string)) | `<list>` | no |
+| pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `<list>` | no |
 | project\_id | The project ID to host the cluster in | string | n/a | yes |
 | region | The region to host the cluster in | string | n/a | yes |
+| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no |
 | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no |
 | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no |
 | subnetwork | The subnetwork to host the cluster in | string | n/a | yes |
+| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `<list>` | no |
 
 ## Outputs
 

examples/simple_regional_beta/main.tf

@@ -24,23 +24,27 @@ provider "google-beta" {
 }
 
 module "gke" {
-  source                   = "../../modules/beta-public-cluster/"
-  project_id               = var.project_id
-  name                     = "${local.cluster_type}-cluster${var.cluster_name_suffix}"
-  regional                 = true
-  region                   = var.region
-  network                  = var.network
-  subnetwork               = var.subnetwork
-  ip_range_pods            = var.ip_range_pods
-  ip_range_services        = var.ip_range_services
-  create_service_account   = false
-  service_account          = var.compute_engine_service_account
-  istio                    = var.istio
-  cloudrun                 = var.cloudrun
-  node_metadata            = var.node_metadata
-  sandbox_enabled          = var.sandbox_enabled
-  remove_default_node_pool = var.remove_default_node_pool
-  node_pools               = var.node_pools
+  source                      = "../../modules/beta-public-cluster/"
+  project_id                  = var.project_id
+  name                        = "${local.cluster_type}-cluster${var.cluster_name_suffix}"
+  regional                    = var.regional
+  region                      = var.region
+  zones                       = var.zones
+  network                     = var.network
+  subnetwork                  = var.subnetwork
+  ip_range_pods               = var.ip_range_pods
+  ip_range_services           = var.ip_range_services
+  create_service_account      = var.compute_engine_service_account == "create"
+  service_account             = var.compute_engine_service_account
+  istio                       = var.istio
+  cloudrun                    = var.cloudrun
+  node_metadata               = var.node_metadata
+  sandbox_enabled             = var.sandbox_enabled
+  remove_default_node_pool    = var.remove_default_node_pool
+  node_pools                  = var.node_pools
+  database_encryption         = var.database_encryption
+  enable_binary_authorization = var.enable_binary_authorization
+  pod_security_policy_config  = var.pod_security_policy_config
 }
 
 data "google_client_config" "default" {

examples/simple_regional_beta/variables.tf

@@ -85,3 +85,36 @@ variable "node_pools" {
     },
   ]
 }
+
+variable "database_encryption" {
+  description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key."
+  type        = list(object({ state = string, key_name = string }))
+  default = [{
+    state    = "DECRYPTED"
+    key_name = ""
+  }]
+}
+
+variable "enable_binary_authorization" {
+  description = "Enable BinAuthZ Admission controller"
+  default     = false
+}
+
+variable "pod_security_policy_config" {
+  description = "enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created."
+  default = [{
+    "enabled" = false
+  }]
+}
+
+variable "zones" {
+  type        = list(string)
+  description = "The zones to host the cluster in (optional if regional cluster / required if zonal)"
+  default     = []
+}
+
+variable "regional" {
+  type        = bool
+  description = "Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!)"
+  default     = true
+}

test/ci/beta-cluster.yml

@@ -0,0 +1,18 @@
+---
+
+platform: linux
+
+inputs:
+- name: pull-request
+  path: terraform-google-kubernetes-engine
+
+run:
+  path: make
+  args: ['test_integration']
+  dir: terraform-google-kubernetes-engine
+
+params:
+  SUITE: "beta-cluster-local"
+  COMPUTE_ENGINE_SERVICE_ACCOUNT: ""
+  REGION: "us-east4"
+  ZONES: '["us-east4-a", "us-east4-b", "us-east4-c"]'

test/fixtures/beta_cluster/main.tf

@@ -0,0 +1,67 @@
+/**
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+locals {
+  name       = "beta-cluster-${random_string.suffix.result}"
+  project_id = var.project_ids[0]
+}
+
+resource "google_kms_key_ring" "db" {
+  location = var.region
+  name     = "${local.name}-db"
+  project  = local.project_id
+}
+
+resource "google_kms_crypto_key" "db" {
+  name     = local.name
+  key_ring = google_kms_key_ring.db.self_link
+}
+
+module "this" {
+  source = "../../../examples/simple_regional_beta"
+
+  cluster_name_suffix            = "-${random_string.suffix.result}"
+  project_id                     = local.project_id
+  regional                       = false
+  region                         = var.region
+  zones                          = slice(var.zones, 0, 1)
+  network                        = google_compute_network.main.name
+  subnetwork                     = google_compute_subnetwork.main.name
+  ip_range_pods                  = google_compute_subnetwork.main.secondary_ip_range[0].range_name
+  ip_range_services              = google_compute_subnetwork.main.secondary_ip_range[1].range_name
+  compute_engine_service_account = "create"
+
+  // Beta features
+  istio = true
+
+  database_encryption = [{
+    state    = "ENCRYPTED"
+    key_name = google_kms_crypto_key.db.self_link
+  }]
+
+  cloudrun = true
+
+  enable_binary_authorization = true
+
+  pod_security_policy_config = [{
+    enabled = true
+  }]
+
+  node_metadata = "EXPOSE"
+}
+
+data "google_client_config" "default" {
+}

test/fixtures/beta_cluster/network.tf

@@ -0,0 +1,45 @@
+/**
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+resource "random_string" "suffix" {
+  length  = 4
+  special = false
+  upper   = false
+}
+
+resource "google_compute_network" "main" {
+  name                    = "cft-gke-test-${random_string.suffix.result}"
+  auto_create_subnetworks = false
+  project                 = local.project_id
+}
+
+resource "google_compute_subnetwork" "main" {
+  name          = "cft-gke-test-${random_string.suffix.result}"
+  ip_cidr_range = "10.0.0.0/17"
+  region        = var.region
+  network       = google_compute_network.main.self_link
+  project       = local.project_id
+
+  secondary_ip_range {
+    range_name    = "cft-gke-test-pods-${random_string.suffix.result}"
+    ip_cidr_range = "192.168.0.0/18"
+  }
+
+  secondary_ip_range {
+    range_name    = "cft-gke-test-services-${random_string.suffix.result}"
+    ip_cidr_range = "192.168.64.0/18"
+  }
+}

test/fixtures/beta_cluster/outputs.tf

@@ -0,0 +1,84 @@
+/**
+ * Copyright 2018 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output "project_id" {
+  value = local.project_id
+}
+
+output "region" {
+  value = module.this.region
+}
+
+output "cluster_name" {
+  description = "Cluster name"
+  value       = module.this.cluster_name
+}
+
+output "network" {
+  value = google_compute_network.main.name
+}
+
+output "subnetwork" {
+  value = google_compute_subnetwork.main.name
+}
+
+output "location" {
+  value = module.this.location
+}
+
+output "ip_range_pods" {
+  description = "The secondary IP range used for pods"
+  value       = google_compute_subnetwork.main.secondary_ip_range[0].range_name
+}
+
+output "ip_range_services" {
+  description = "The secondary IP range used for services"
+  value       = google_compute_subnetwork.main.secondary_ip_range[1].range_name
+}
+
+output "zones" {
+  description = "List of zones in which the cluster resides"
+  value       = module.this.zones
+}
+
+output "master_kubernetes_version" {
+  description = "The master Kubernetes version"
+  value       = module.this.master_kubernetes_version
+}
+
+output "kubernetes_endpoint" {
+  sensitive = true
+  value     = module.this.kubernetes_endpoint
+}
+
+output "client_token" {
+  sensitive = true
+  value     = base64encode(data.google_client_config.default.access_token)
+}
+
+output "ca_certificate" {
+  description = "The cluster CA certificate"
+  value       = module.this.ca_certificate
+}
+
+output "service_account" {
+  description = "The service account to default running nodes as if not overridden in `node_pools`."
+  value       = module.this.service_account
+}
+
+output "database_encryption_key_name" {
+  value = google_kms_crypto_key.db.self_link
+}

test/fixtures/beta_cluster/variables.tf

@@ -0,0 +1 @@
+../deploy_service/variables.tf