From fa9b2932b1e11e6f8e2a46340b3bb2d475cbc8a4 Mon Sep 17 00:00:00 2001 From: Nathan McKinley Date: Thu, 20 Dec 2018 13:15:03 -0800 Subject: [PATCH] third party change - Add local ssds and boot disk types to dataproc (#1078) Merged PR #1078. --- build/terraform | 2 +- build/terraform-beta | 2 +- .../resources/resource_dataproc_cluster.go | 26 ++++++++++++++++--- .../tests/resource_dataproc_cluster_test.go | 4 +++ .../docs/r/dataproc_cluster.html.markdown | 8 ++++++ 5 files changed, 36 insertions(+), 6 deletions(-) diff --git a/build/terraform b/build/terraform index 31ba8873c783..222a47a0c384 160000 --- a/build/terraform +++ b/build/terraform @@ -1 +1 @@ -Subproject commit 31ba8873c783c66dff4184519ade25a131ce5313 +Subproject commit 222a47a0c3848c1573b4d10af2858236b63ee8f2 diff --git a/build/terraform-beta b/build/terraform-beta index 5b50cfffefe6..17cea49008aa 160000 --- a/build/terraform-beta +++ b/build/terraform-beta @@ -1 +1 @@ -Subproject commit 5b50cfffefe69feee22af28b8dba19539bdb117f +Subproject commit 17cea49008aa3bcebacf874458397625b563658a diff --git a/third_party/terraform/resources/resource_dataproc_cluster.go b/third_party/terraform/resources/resource_dataproc_cluster.go index 4bcac73b1060..6d38c2f72f75 100644 --- a/third_party/terraform/resources/resource_dataproc_cluster.go +++ b/third_party/terraform/resources/resource_dataproc_cluster.go @@ -215,10 +215,12 @@ func resourceDataprocCluster() *schema.Resource { Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ - - // API does not honour this if set ... - // It simply ignores it completely - // "num_local_ssds": { ... } + "num_local_ssds": { + Type: schema.TypeInt, + Optional: true, + Computed: true, + ForceNew: true, + }, "boot_disk_size_gb": { Type: schema.TypeInt, @@ -227,6 +229,14 @@ func resourceDataprocCluster() *schema.Resource { ForceNew: true, ValidateFunc: validation.IntAtLeast(10), }, + + "boot_disk_type": { + Type: schema.TypeString, + Optional: true, + ForceNew: true, + ValidateFunc: validation.StringInSlice([]string{"pd-standard", "pd-ssd", ""}, false), + Default: "pd-standard", + }, }, }, }, @@ -611,6 +621,12 @@ func expandPreemptibleInstanceGroupConfig(cfg map[string]interface{}) *dataproc. if v, ok := dcfg["boot_disk_size_gb"]; ok { icg.DiskConfig.BootDiskSizeGb = int64(v.(int)) } + if v, ok := dcfg["num_local_ssds"]; ok { + icg.DiskConfig.NumLocalSsds = int64(v.(int)) + } + if v, ok := dcfg["boot_disk_type"]; ok { + icg.DiskConfig.BootDiskType = v.(string) + } } } return icg @@ -869,6 +885,8 @@ func flattenPreemptibleInstanceGroupConfig(d *schema.ResourceData, icg *dataproc data["instance_names"] = icg.InstanceNames if icg.DiskConfig != nil { disk["boot_disk_size_gb"] = icg.DiskConfig.BootDiskSizeGb + disk["num_local_ssds"] = icg.DiskConfig.NumLocalSsds + disk["boot_disk_type"] = icg.DiskConfig.BootDiskType } } diff --git a/third_party/terraform/tests/resource_dataproc_cluster_test.go b/third_party/terraform/tests/resource_dataproc_cluster_test.go index c8be786bcfca..c0c3a493fbd2 100644 --- a/third_party/terraform/tests/resource_dataproc_cluster_test.go +++ b/third_party/terraform/tests/resource_dataproc_cluster_test.go @@ -610,6 +610,8 @@ func validateDataprocCluster_withConfigOverrides(n string, cluster *dataproc.Clu {"cluster_config.0.preemptible_worker_config.0.num_instances", "1", strconv.Itoa(int(cluster.Config.SecondaryWorkerConfig.NumInstances))}, {"cluster_config.0.preemptible_worker_config.0.disk_config.0.boot_disk_size_gb", "12", strconv.Itoa(int(cluster.Config.SecondaryWorkerConfig.DiskConfig.BootDiskSizeGb))}, + {"cluster_config.0.preemptible_worker_config.0.disk_config.0.num_local_ssds", "1", strconv.Itoa(int(cluster.Config.SecondaryWorkerConfig.DiskConfig.NumLocalSsds))}, + {"cluster_config.0.preemptible_worker_config.0.disk_config.0.boot_disk_type", "pd-ssd", cluster.Config.SecondaryWorkerConfig.DiskConfig.BootDiskType}, {"cluster_config.0.preemptible_worker_config.0.instance_names.#", "1", strconv.Itoa(len(cluster.Config.SecondaryWorkerConfig.InstanceNames))}, } @@ -853,7 +855,9 @@ resource "google_dataproc_cluster" "with_config_overrides" { preemptible_worker_config { num_instances = 1 disk_config { + boot_disk_type = "pd-ssd" boot_disk_size_gb = 12 + num_local_ssds = 1 } } } diff --git a/third_party/terraform/website/docs/r/dataproc_cluster.html.markdown b/third_party/terraform/website/docs/r/dataproc_cluster.html.markdown index ba1d6ae64b81..fafec09f165c 100644 --- a/third_party/terraform/website/docs/r/dataproc_cluster.html.markdown +++ b/third_party/terraform/website/docs/r/dataproc_cluster.html.markdown @@ -343,7 +343,9 @@ The `cluster_config.preemptible_worker_config` block supports: preemptible_worker_config { num_instances = 1 disk_config { + boot_disk_type = "pd-standard" boot_disk_size_gb = 10 + num_local_ssds = 1 } } } @@ -357,11 +359,17 @@ will be set for you based on whatever was set for the `worker_config.machine_typ * `disk_config` (Optional) Disk Config + * `boot_disk_type` - (Optional) The disk type of the primary disk attached to each preemptible worker node. + One of `"pd-ssd"` or `"pd-standard"`. Defaults to `"pd-standard"`. + * `boot_disk_size_gb` - (Optional, Computed) Size of the primary disk attached to each preemptible worker node, specified in GB. The smallest allowed disk size is 10GB. GCP will default to a predetermined computed value if not set (currently 500GB). Note: If SSDs are not attached, it also contains the HDFS data blocks and Hadoop working directories. + * `num_local_ssds` - (Optional) The amount of local SSD disks that will be + attached to each preemptible worker node. Defaults to 0. + - - - The `cluster_config.software_config` block supports: