From bfaf44f922a958f9b8a7c0d82eb3b17269cbe70d Mon Sep 17 00:00:00 2001 From: Arthur Burkart Date: Sun, 8 Dec 2019 14:02:00 +0100 Subject: [PATCH 1/2] resource/aws_emr_cluster: Implements step_concurrency_level * Please vote on this pull request by adding a +1 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to the original pull request comment to help the community and maintainers prioritize this request * Please do not leave "+1" comments, they generate extra noise for pull request followers and do not help prioritize the request Closes #11159 Release note for [CHANGELOG](https://github.com/terraform-providers/terraform-provider-aws/blob/master/CHANGELOG.md): ```md * resource/aws_emr_cluster: Support step concurrency level ``` Output from acceptance testing: ``` --- PASS: TestAccAWSEMRCluster_step_concurrency_level (538.64s) ``` --- aws/resource_aws_emr_cluster.go | 22 +++++++++ aws/resource_aws_emr_cluster_test.go | 60 ++++++++++++++++++++++++ website/docs/r/emr_cluster.html.markdown | 1 + 3 files changed, 83 insertions(+) diff --git a/aws/resource_aws_emr_cluster.go b/aws/resource_aws_emr_cluster.go index 4a2306c2860..bcbda7cc361 100644 --- a/aws/resource_aws_emr_cluster.go +++ b/aws/resource_aws_emr_cluster.go @@ -630,6 +630,11 @@ func resourceAwsEMRCluster() *schema.Resource { Optional: true, ValidateFunc: validateAwsEmrCustomAmiId, }, + "step_concurrency_level": { + Type: schema.TypeInt, + Optional: true, + ValidateFunc: validation.IntBetween(1, 256), + }, }, } } @@ -842,6 +847,10 @@ func resourceAwsEMRClusterCreate(d *schema.ResourceData, meta interface{}) error params.CustomAmiId = aws.String(v.(string)) } + if v, ok := d.GetOk("step_concurrency_level"); ok { + params.StepConcurrencyLevel = aws.Int64(int64(v.(int))) + } + if instanceProfile != "" { params.JobFlowRole = aws.String(instanceProfile) } @@ -1040,6 +1049,7 @@ func resourceAwsEMRClusterRead(d *schema.ResourceData, meta interface{}) error { d.Set("ebs_root_volume_size", cluster.EbsRootVolumeSize) d.Set("scale_down_behavior", cluster.ScaleDownBehavior) d.Set("termination_protection", cluster.TerminationProtected) + d.Set("step_concurrency_level", cluster.StepConcurrencyLevel) if cluster.CustomAmiId != nil { d.Set("custom_ami_id", cluster.CustomAmiId) @@ -1339,6 +1349,18 @@ func resourceAwsEMRClusterUpdate(d *schema.ResourceData, meta interface{}) error } } + if d.HasChange("step_concurrency_level") { + d.SetPartial("step_concurrency_level") + _, errModify := conn.ModifyCluster(&emr.ModifyClusterInput{ + ClusterId: aws.String(d.Id()), + StepConcurrencyLevel: aws.Int64(int64(d.Get("step_concurrency_level").(int))), + }) + if errModify != nil { + log.Printf("[ERROR] %s", errModify) + return errModify + } + } + d.Partial(false) return resourceAwsEMRClusterRead(d, meta) diff --git a/aws/resource_aws_emr_cluster_test.go b/aws/resource_aws_emr_cluster_test.go index 392afdc88cd..30dc035f857 100644 --- a/aws/resource_aws_emr_cluster_test.go +++ b/aws/resource_aws_emr_cluster_test.go @@ -1363,6 +1363,39 @@ func TestAccAWSEMRCluster_root_volume_size(t *testing.T) { }) } +func TestAccAWSEMRCluster_step_concurrency_level(t *testing.T) { + var cluster emr.Cluster + rName := acctest.RandomWithPrefix("tf-acc-test") + resourceName := "aws_emr_cluster.tf-test-cluster" + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSEmrDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAWSEmrClusterConfigStepConcurrencyLevel(rName, 2), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSEmrClusterExists(resourceName, &cluster), + resource.TestCheckResourceAttr(resourceName, "step_concurrency_level", "2"), + ), + }, + { + Config: testAccAWSEmrClusterConfigStepConcurrencyLevel(rName, 1), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSEmrClusterExists(resourceName, &cluster), + resource.TestCheckResourceAttr(resourceName, "step_concurrency_level", "1"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"configurations", "keep_job_flow_alive_when_no_steps"}, + }, + }, + }) +} + func TestAccAWSEMRCluster_custom_ami_id(t *testing.T) { var cluster emr.Cluster r := acctest.RandInt() @@ -7616,3 +7649,30 @@ data "aws_ami" "emr-custom-ami" { } `, r, r, r, r, r, r, r, r) } + +func testAccAWSEmrClusterConfigStepConcurrencyLevel(rName string, stepConcurrencyLevel int) string { + return testAccAWSEmrClusterConfigBaseVpc(false) + fmt.Sprintf(` +resource "aws_emr_cluster" "tf-test-cluster" { + applications = ["Spark"] + keep_job_flow_alive_when_no_steps = true + name = %[1]q + release_label = "emr-5.28.0" + service_role = "EMR_DefaultRole" + + ec2_attributes { + emr_managed_master_security_group = "${aws_security_group.test.id}" + emr_managed_slave_security_group = "${aws_security_group.test.id}" + instance_profile = "EMR_EC2_DefaultRole" + subnet_id = "${aws_subnet.test.id}" + } + + master_instance_group { + instance_type = "m4.large" + } + + step_concurrency_level = %[2]d + + depends_on = ["aws_route_table_association.test"] +} +`, rName, stepConcurrencyLevel) +} diff --git a/website/docs/r/emr_cluster.html.markdown b/website/docs/r/emr_cluster.html.markdown index ac56234186a..a6c49ea33b7 100644 --- a/website/docs/r/emr_cluster.html.markdown +++ b/website/docs/r/emr_cluster.html.markdown @@ -274,6 +274,7 @@ EOF * `visible_to_all_users` - (Optional) Whether the job flow is visible to all IAM users of the AWS account associated with the job flow. Default `true` * `autoscaling_role` - (Optional) An IAM role for automatic scaling policies. The IAM role provides permissions that the automatic scaling feature requires to launch and terminate EC2 instances in an instance group. * `step` - (Optional) List of steps to run when creating the cluster. Defined below. It is highly recommended to utilize the [lifecycle configuration block](/docs/configuration/resources.html) with `ignore_changes` if other steps are being managed outside of Terraform. This argument is processed in [attribute-as-blocks mode](/docs/configuration/attr-as-blocks.html). +* `step_concurrency_level` - (Optional) The number of steps that can be executed concurrently. You can specify a maximum of 256 steps. Only valid for EMR clusters with `release_label` 5.28.0 or greater. (default is 1) * `tags` - (Optional) list of tags to apply to the EMR Cluster ## core_instance_group Configuration Block From 8df20396ec697c265d7ce07894fa1c7141894acc Mon Sep 17 00:00:00 2001 From: Arthur Burkart Date: Mon, 9 Dec 2019 22:14:20 +0100 Subject: [PATCH 2/2] resource/aws_emr_cluster: step concurrency level defaults to 1 --- aws/resource_aws_emr_cluster.go | 1 + 1 file changed, 1 insertion(+) diff --git a/aws/resource_aws_emr_cluster.go b/aws/resource_aws_emr_cluster.go index bcbda7cc361..de8e184447d 100644 --- a/aws/resource_aws_emr_cluster.go +++ b/aws/resource_aws_emr_cluster.go @@ -633,6 +633,7 @@ func resourceAwsEMRCluster() *schema.Resource { "step_concurrency_level": { Type: schema.TypeInt, Optional: true, + Default: 1, ValidateFunc: validation.IntBetween(1, 256), }, },