add auto_provisioning_defaults to `google_container_cluster.cluster…

…_autoscaling` Signed-off-by: Modular Magician <[email protected]>
hashicorp · Nov 26, 2019 · c986a5f · c986a5f
1 parent 08ef6ce
commit c986a5f
Show file tree

Hide file tree

Showing 6 changed files with 268 additions and 4 deletions.
diff --git a/google/node_config.go b/google/node_config.go
@@ -120,7 +120,8 @@ var schemaNodeConfig = &schema.Schema{
 						return canonicalizeServiceScope(v.(string))
 					},
 				},
-				Set: stringScopeHashcode,
+				DiffSuppressFunc: containerClusterAddedScopesSuppress,
+				Set:              stringScopeHashcode,
 			},
 
 			"preemptible": {

diff --git a/google/resource_container_cluster.go b/google/resource_container_cluster.go
@@ -215,7 +215,9 @@ func resourceContainerCluster() *schema.Resource {
 			"cluster_autoscaling": {
 				Type:     schema.TypeList,
 				MaxItems: 1,
-				Removed:  "This field is in beta. Use it in the the google-beta provider instead. See https://terraform.io/docs/providers/google/guides/provider_versions.html for more details.",
+				// This field is Optional + Computed because we automatically set the
+				// enabled value to false if the block is not returned in API responses.
+				Optional: true,
 				Computed: true,
 				Elem: &schema.Resource{
 					Schema: map[string]*schema.Schema{
@@ -243,6 +245,34 @@ func resourceContainerCluster() *schema.Resource {
 								},
 							},
 						},
+						"auto_provisioning_defaults": {
+							Type:     schema.TypeList,
+							MaxItems: 1,
+							Optional: true,
+							Elem: &schema.Resource{
+								Schema: map[string]*schema.Schema{
+									"oauth_scopes": {
+										Type:             schema.TypeList,
+										Optional:         true,
+										Elem:             &schema.Schema{Type: schema.TypeString},
+										DiffSuppressFunc: containerClusterAddedScopesSuppress,
+										ExactlyOneOf: []string{
+											"cluster_autoscaling.0.auto_provisioning_defaults.0.oauth_scopes",
+											"cluster_autoscaling.0.auto_provisioning_defaults.0.service_account",
+										},
+									},
+									"service_account": {
+										Type:     schema.TypeString,
+										Optional: true,
+										Default:  "default",
+										ExactlyOneOf: []string{
+											"cluster_autoscaling.0.auto_provisioning_defaults.0.oauth_scopes",
+											"cluster_autoscaling.0.auto_provisioning_defaults.0.service_account",
+										},
+									},
+								},
+							},
+						},
 					},
 				},
 			},
@@ -762,6 +792,7 @@ func resourceContainerClusterCreate(d *schema.ResourceData, meta interface{}) er
 		EnableKubernetesAlpha:   d.Get("enable_kubernetes_alpha").(bool),
 		IpAllocationPolicy:      expandIPAllocationPolicy(d.Get("ip_allocation_policy")),
 		PodSecurityPolicyConfig: expandPodSecurityPolicyConfig(d.Get("pod_security_policy_config")),
+		Autoscaling:             expandClusterAutoscaling(d.Get("cluster_autoscaling"), d),
 		MasterAuth:              expandMasterAuth(d.Get("master_auth")),
 		ResourceLabels:          expandStringMap(d, "resource_labels"),
 	}
@@ -966,7 +997,7 @@ func resourceContainerClusterRead(d *schema.ResourceData, meta interface{}) erro
 	d.Set("monitoring_service", cluster.MonitoringService)
 	d.Set("network", cluster.NetworkConfig.Network)
 	d.Set("subnetwork", cluster.NetworkConfig.Subnetwork)
-	if err := d.Set("cluster_autoscaling", nil); err != nil {
+	if err := d.Set("cluster_autoscaling", flattenClusterAutoscaling(cluster.Autoscaling)); err != nil {
 		return err
 	}
 	if err := d.Set("authenticator_groups_config", flattenAuthenticatorGroupsConfig(cluster.AuthenticatorGroupsConfig)); err != nil {
@@ -1087,6 +1118,23 @@ func resourceContainerClusterUpdate(d *schema.ResourceData, meta interface{}) er
 		}
 	}
 
+	if d.HasChange("cluster_autoscaling") {
+		req := &containerBeta.UpdateClusterRequest{
+			Update: &containerBeta.ClusterUpdate{
+				DesiredClusterAutoscaling: expandClusterAutoscaling(d.Get("cluster_autoscaling"), d),
+			}}
+
+		updateF := updateFunc(req, "updating GKE cluster autoscaling")
+		// Call update serially.
+		if err := lockedCall(lockKey, updateF); err != nil {
+			return err
+		}
+
+		log.Printf("[INFO] GKE cluster %s's cluster-wide autoscaling has been updated", d.Id())
+
+		d.SetPartial("cluster_autoscaling")
+	}
+
 	if d.HasChange("maintenance_policy") {
 		req := &containerBeta.SetMaintenancePolicyRequest{
 			MaintenancePolicy: expandMaintenancePolicy(d, meta),
@@ -1691,6 +1739,59 @@ func expandMaintenancePolicy(d *schema.ResourceData, meta interface{}) *containe
 	return nil
 }
 
+func expandClusterAutoscaling(configured interface{}, d *schema.ResourceData) *containerBeta.ClusterAutoscaling {
+	l, ok := configured.([]interface{})
+	if !ok || l == nil || len(l) == 0 || l[0] == nil {
+		return &containerBeta.ClusterAutoscaling{
+			EnableNodeAutoprovisioning: false,
+			ForceSendFields:            []string{"EnableNodeAutoprovisioning"},
+		}
+	}
+
+	config := l[0].(map[string]interface{})
+
+	// Conditionally provide an empty list to preserve a legacy 2.X behaviour
+	// when `enabled` is false and resource_limits is unset, allowing users to
+	// explicitly disable the feature. resource_limits don't work when node
+	// auto-provisioning is disabled at time of writing. This may change API-side
+	// in the future though, as the feature is intended to apply to both node
+	// auto-provisioning and node autoscaling.
+	var resourceLimits []*containerBeta.ResourceLimit
+	if limits, ok := config["resource_limits"]; ok {
+		resourceLimits = make([]*containerBeta.ResourceLimit, 0)
+		if lmts, ok := limits.([]interface{}); ok {
+			for _, v := range lmts {
+				limit := v.(map[string]interface{})
+				resourceLimits = append(resourceLimits,
+					&containerBeta.ResourceLimit{
+						ResourceType: limit["resource_type"].(string),
+						// Here we're relying on *not* setting ForceSendFields for 0-values.
+						Minimum: int64(limit["minimum"].(int)),
+						Maximum: int64(limit["maximum"].(int)),
+					})
+			}
+		}
+	}
+	return &containerBeta.ClusterAutoscaling{
+		EnableNodeAutoprovisioning:       config["enabled"].(bool),
+		ResourceLimits:                   resourceLimits,
+		AutoprovisioningNodePoolDefaults: expandAutoProvisioningDefaults(config["auto_provisioning_defaults"], d),
+	}
+}
+
+func expandAutoProvisioningDefaults(configured interface{}, d *schema.ResourceData) *containerBeta.AutoprovisioningNodePoolDefaults {
+	l, ok := configured.([]interface{})
+	if !ok || l == nil || len(l) == 0 || l[0] == nil {
+		return &containerBeta.AutoprovisioningNodePoolDefaults{}
+	}
+	config := l[0].(map[string]interface{})
+
+	return &containerBeta.AutoprovisioningNodePoolDefaults{
+		OauthScopes:    convertStringArr(config["oauth_scopes"].([]interface{})),
+		ServiceAccount: config["service_account"].(string),
+	}
+}
+
 func expandAuthenticatorGroupsConfig(configured interface{}) *containerBeta.AuthenticatorGroupsConfig {
 	l := configured.([]interface{})
 	if len(l) == 0 {
@@ -1954,6 +2055,34 @@ func flattenMasterAuth(ma *containerBeta.MasterAuth) []map[string]interface{} {
 	return masterAuth
 }
 
+func flattenClusterAutoscaling(a *containerBeta.ClusterAutoscaling) []map[string]interface{} {
+	r := make(map[string]interface{})
+	if a == nil || !a.EnableNodeAutoprovisioning {
+		r["enabled"] = false
+	} else {
+		resourceLimits := make([]interface{}, 0, len(a.ResourceLimits))
+		for _, rl := range a.ResourceLimits {
+			resourceLimits = append(resourceLimits, map[string]interface{}{
+				"resource_type": rl.ResourceType,
+				"minimum":       rl.Minimum,
+				"maximum":       rl.Maximum,
+			})
+		}
+		r["resource_limits"] = resourceLimits
+		r["enabled"] = true
+		r["auto_provisioning_defaults"] = flattenAutoProvisioningDefaults(a.AutoprovisioningNodePoolDefaults)
+	}
+	return []map[string]interface{}{r}
+}
+
+func flattenAutoProvisioningDefaults(a *containerBeta.AutoprovisioningNodePoolDefaults) []map[string]interface{} {
+	r := make(map[string]interface{})
+	r["oauth_scopes"] = a.OauthScopes
+	r["service_account"] = a.ServiceAccount
+
+	return []map[string]interface{}{r}
+}
+
 func flattenMasterAuthorizedNetworksConfig(c *containerBeta.MasterAuthorizedNetworksConfig) []map[string]interface{} {
 	if c == nil || !c.Enabled {
 		return nil
@@ -2031,6 +2160,34 @@ func cidrOrSizeDiffSuppress(k, old, new string, d *schema.ResourceData) bool {
 	return strings.HasPrefix(new, "/") && strings.HasSuffix(old, new)
 }
 
+// We want to suppress diffs for the scopes automatically added by GKE
+func containerClusterAddedScopesSuppress(k, old, new string, d *schema.ResourceData) bool {
+	o, n := d.GetChange("cluster_autoscaling.0.auto_provisioning_defaults.0.oauth_scopes")
+
+	addedScopes := []string{
+		"https://www.googleapis.com/auth/monitoring.write",
+	}
+
+	// combine what the default scopes are with what was passed
+	m := golangSetFromStringSlice(append(addedScopes, convertStringArr(n.([]interface{}))...))
+	combined := stringSliceFromGolangSet(m)
+
+	// compare if the combined new scopes and default scopes differ from the old scopes
+	if len(combined) != len(convertStringArr(o.([]interface{}))) {
+		return false
+	}
+
+	for _, i := range combined {
+		if stringInSlice(convertStringArr(o.([]interface{})), i) {
+			continue
+		}
+
+		return false
+	}
+
+	return true
+}
+
 // We want to suppress diffs for empty/disabled private cluster config.
 func containerClusterPrivateClusterConfigSuppress(k, old, new string, d *schema.ResourceData) bool {
 	o, n := d.GetChange("private_cluster_config.0.enable_private_endpoint")

diff --git a/google/resource_container_cluster_test.go b/google/resource_container_cluster_test.go
@@ -955,6 +955,46 @@ func TestAccContainerCluster_withIPAllocationPolicy_specificSizes(t *testing.T)
 	})
 }
 
+func TestAccContainerCluster_nodeAutoprovisioning(t *testing.T) {
+	t.Parallel()
+
+	clusterName := fmt.Sprintf("cluster-test-%s", acctest.RandString(10))
+
+	resource.Test(t, resource.TestCase{
+		PreCheck:     func() { testAccPreCheck(t) },
+		Providers:    testAccProviders,
+		CheckDestroy: testAccCheckContainerClusterDestroy,
+		Steps: []resource.TestStep{
+			{
+				Config: testAccContainerCluster_autoprovisioning(clusterName, true),
+				Check: resource.ComposeTestCheckFunc(
+					resource.TestCheckResourceAttr("google_container_cluster.with_autoprovisioning",
+						"cluster_autoscaling.0.enabled", "true"),
+				),
+			},
+			{
+				ResourceName:            "google_container_cluster.with_autoprovisioning",
+				ImportState:             true,
+				ImportStateVerify:       true,
+				ImportStateVerifyIgnore: []string{"min_master_version"},
+			},
+			{
+				Config: testAccContainerCluster_autoprovisioning(clusterName, false),
+				Check: resource.ComposeTestCheckFunc(
+					resource.TestCheckResourceAttr("google_container_cluster.with_autoprovisioning",
+						"cluster_autoscaling.0.enabled", "false"),
+				),
+			},
+			{
+				ResourceName:            "google_container_cluster.with_autoprovisioning",
+				ImportState:             true,
+				ImportStateVerify:       true,
+				ImportStateVerifyIgnore: []string{"min_master_version"},
+			},
+		},
+	})
+}
+
 func TestAccContainerCluster_errorCleanDanglingCluster(t *testing.T) {
 	t.Parallel()
 
@@ -1909,6 +1949,52 @@ resource "google_container_cluster" "with_node_pool" {
 `, cluster, nodePool)
 }
 
+func testAccContainerCluster_autoprovisioning(cluster string, autoprovisioning bool) string {
+	config := fmt.Sprintf(`
+data "google_container_engine_versions" "central1a" {
+  location = "us-central1-a"
+}
+
+resource "google_container_cluster" "with_autoprovisioning" {
+  name               = "%s"
+  location           = "us-central1-a"
+  min_master_version = data.google_container_engine_versions.central1a.latest_master_version
+  initial_node_count = 1
+
+  logging_service    = "none"
+  monitoring_service = "none"
+
+`, cluster)
+	if autoprovisioning {
+		config += `
+  cluster_autoscaling {
+    enabled = true
+    resource_limits {
+      resource_type = "cpu"
+      maximum       = 2
+    }
+    resource_limits {
+      resource_type = "memory"
+      maximum       = 2048
+    }
+    auto_provisioning_defaults {
+      oauth_scopes = [
+        "https://www.googleapis.com/auth/pubsub",
+        "https://www.googleapis.com/auth/devstorage.read_only"
+      ]
+    }
+  }`
+	} else {
+		config += `
+  cluster_autoscaling {
+    enabled = false
+  }`
+	}
+	config += `
+}`
+	return config
+}
+
 func testAccContainerCluster_withNodePoolAutoscaling(cluster, np string) string {
 	return fmt.Sprintf(`
 resource "google_container_cluster" "with_node_pool" {

diff --git a/google/resource_sql_database_instance_test.go b/google/resource_sql_database_instance_test.go
@@ -9,7 +9,6 @@ import (
 	"github.com/hashicorp/terraform-plugin-sdk/helper/acctest"
 	"github.com/hashicorp/terraform-plugin-sdk/helper/resource"
 	"github.com/hashicorp/terraform-plugin-sdk/terraform"
-
 	sqladmin "google.golang.org/api/sqladmin/v1beta4"
 )
 

diff --git a/google/utils.go b/google/utils.go
@@ -565,3 +565,13 @@ func calcAddRemove(from []string, to []string) (add, remove []string) {
 	}
 	return add, remove
 }
+
+func stringInSlice(arr []string, str string) bool {
+	for _, i := range arr {
+		if i == str {
+			return true
+		}
+	}
+
+	return false
+}
diff --git a/website/docs/r/container_cluster.html.markdown b/website/docs/r/container_cluster.html.markdown
@@ -362,6 +362,9 @@ cluster. Configuring the `cpu` and `memory` types is required if node
 auto-provisioning is enabled. These limits will apply to node pool autoscaling
 in addition to node auto-provisioning. Structure is documented below.
 
+* `auto_provisioning_defaults` - (Optional) Contains defaults for a node pool created by NAP.
+Structure is documented below.
+
 The `resource_limits` block supports:
 
 * `resource_type` - (Required) The type of the resource. For example, `cpu` and
@@ -372,6 +375,14 @@ for a list of types.
 
 * `maximum` - (Optional) Maximum amount of the resource in the cluster.
 
+The `auto_provisioning_defaults` block supports:
+
+* `oauth_scopes` - (Optional) Scopes that are used by NAP when creating node pools.
+If `oauth_scopes` are specified, `service_account` must be empty.
+
+* `service_account` - (Optional) The Google Cloud Platform Service Account to be used by the node VMs.
+If `service_account` is specified, `oauth_scopes` must be empty.
+
 The `authenticator_groups_config` block supports:
 
 * `security_group` - (Required) The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format `[email protected]`.