#713 Support autoscaling on EMR

aws/resource_aws_emr_cluster.go

@@ -15,6 +15,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/emr"
 	"github.com/hashicorp/terraform/helper/resource"
 	"github.com/hashicorp/terraform/helper/schema"
+	"github.com/hashicorp/terraform/helper/structure"
 )
 
 func resourceAwsEMRCluster() *schema.Resource {
@@ -173,6 +174,16 @@ func resourceAwsEMRCluster() *schema.Resource {
 							Optional: true,
 							Default:  0,
 						},
+						"autoscaling_policy": {
+							Type:             schema.TypeString,
+							Optional:         true,
+							DiffSuppressFunc: suppressEquivalentJsonDiffs,
+							ValidateFunc:     validateJsonString,
+							StateFunc: func(v interface{}) string {
+								jsonString, _ := structure.NormalizeJsonString(v)
+								return jsonString
+							},
+						},
 						"instance_role": {
 							Type:         schema.TypeString,
 							Required:     true,
@@ -750,6 +761,13 @@ func flattenInstanceGroups(igs []*emr.InstanceGroup) []map[string]interface{} {
 		attrs["instance_count"] = *ig.RequestedInstanceCount
 		attrs["instance_role"] = *ig.InstanceGroupType
 		attrs["instance_type"] = *ig.InstanceType
+
+		if ig.AutoScalingPolicy != nil {
+			attrs["autoscaling_policy"] = *ig.AutoScalingPolicy
+		} else {
+			attrs["autoscaling_policy"] = ""
+		}
+
 		attrs["name"] = *ig.Name
 		result = append(result, attrs)
 	}
@@ -897,7 +915,7 @@ func expandBootstrapActions(bootstrapActions []interface{}) []*emr.BootstrapActi
 }
 
 func expandInstanceGroupConfigs(instanceGroupConfigs []interface{}) []*emr.InstanceGroupConfig {
-	configsOut := []*emr.InstanceGroupConfig{}
+	instanceGroupConfig := []*emr.InstanceGroupConfig{}
 
 	for _, raw := range instanceGroupConfigs {
 		configAttributes := raw.(map[string]interface{})
@@ -912,42 +930,68 @@ func expandInstanceGroupConfigs(instanceGroupConfigs []interface{}) []*emr.Insta
 			InstanceCount: aws.Int64(int64(configInstanceCount)),
 		}
 
-		if bidPrice, ok := configAttributes["bid_price"]; ok {
-			if bidPrice != "" {
-				config.BidPrice = aws.String(bidPrice.(string))
-				config.Market = aws.String("SPOT")
-			} else {
-				config.Market = aws.String("ON_DEMAND")
-			}
+		applyBidPrice(config, configAttributes)
+		applyEbsConfig(configAttributes, config)
+		applyAutoScalingPolicy(configAttributes, config)
+
+		instanceGroupConfig = append(instanceGroupConfig, config)
+	}
+
+	return instanceGroupConfig
+}
+
+func applyBidPrice(config *emr.InstanceGroupConfig, configAttributes map[string]interface{}) {
+	if bidPrice, ok := configAttributes["bid_price"]; ok {
+		if bidPrice != "" {
+			config.BidPrice = aws.String(bidPrice.(string))
+			config.Market = aws.String("SPOT")
+		} else {
+			config.Market = aws.String("ON_DEMAND")
 		}
+	}
+}
 
-		if rawEbsConfigs, ok := configAttributes["ebs_config"]; ok {
-			ebsConfig := &emr.EbsConfiguration{}
-
-			ebsBlockDeviceConfigs := make([]*emr.EbsBlockDeviceConfig, 0)
-			for _, rawEbsConfig := range rawEbsConfigs.(*schema.Set).List() {
-				rawEbsConfig := rawEbsConfig.(map[string]interface{})
-				ebsBlockDeviceConfig := &emr.EbsBlockDeviceConfig{
-					VolumesPerInstance: aws.Int64(int64(rawEbsConfig["volumes_per_instance"].(int))),
-					VolumeSpecification: &emr.VolumeSpecification{
-						SizeInGB:   aws.Int64(int64(rawEbsConfig["size"].(int))),
-						VolumeType: aws.String(rawEbsConfig["type"].(string)),
-					},
-				}
-				if v, ok := rawEbsConfig["iops"].(int); ok && v != 0 {
-					ebsBlockDeviceConfig.VolumeSpecification.Iops = aws.Int64(int64(v))
-				}
-				ebsBlockDeviceConfigs = append(ebsBlockDeviceConfigs, ebsBlockDeviceConfig)
+func applyEbsConfig(configAttributes map[string]interface{}, config *emr.InstanceGroupConfig) {
+	if rawEbsConfigs, ok := configAttributes["ebs_config"]; ok {
+		ebsConfig := &emr.EbsConfiguration{}
+
+		ebsBlockDeviceConfigs := make([]*emr.EbsBlockDeviceConfig, 0)
+		for _, rawEbsConfig := range rawEbsConfigs.(*schema.Set).List() {
+			rawEbsConfig := rawEbsConfig.(map[string]interface{})
+			ebsBlockDeviceConfig := &emr.EbsBlockDeviceConfig{
+				VolumesPerInstance: aws.Int64(int64(rawEbsConfig["volumes_per_instance"].(int))),
+				VolumeSpecification: &emr.VolumeSpecification{
+					SizeInGB:   aws.Int64(int64(rawEbsConfig["size"].(int))),
+					VolumeType: aws.String(rawEbsConfig["type"].(string)),
+				},
 			}
-			ebsConfig.EbsBlockDeviceConfigs = ebsBlockDeviceConfigs
-
-			config.EbsConfiguration = ebsConfig
+			if v, ok := rawEbsConfig["iops"].(int); ok && v != 0 {
+				ebsBlockDeviceConfig.VolumeSpecification.Iops = aws.Int64(int64(v))
+			}
+			ebsBlockDeviceConfigs = append(ebsBlockDeviceConfigs, ebsBlockDeviceConfig)
 		}
+		ebsConfig.EbsBlockDeviceConfigs = ebsBlockDeviceConfigs
 
-		configsOut = append(configsOut, config)
+		config.EbsConfiguration = ebsConfig
 	}
+}
 
-	return configsOut
+func applyAutoScalingPolicy(configAttributes map[string]interface{}, config *emr.InstanceGroupConfig) {
+	if rawAutoScalingPolicy, ok := configAttributes["autoscaling_policy"]; ok {
+		autoScalingConfig, _ := expandAutoScalingPolicy(rawAutoScalingPolicy.(string))
+		config.AutoScalingPolicy = autoScalingConfig
+	}
+}
+
+func expandAutoScalingPolicy(rawDefinitions string) (*emr.AutoScalingPolicy, error) {
+	var policy *emr.AutoScalingPolicy
+
+	err := json.Unmarshal([]byte(rawDefinitions), &policy)
+	if err != nil {
+		return nil, fmt.Errorf("Error decoding JSON: %s", err)
+	}
+
+	return policy, nil
 }
 
 func expandConfigures(input string) []*emr.Configuration {

aws/resource_aws_emr_cluster_test.go

@@ -1365,6 +1365,39 @@ resource "aws_emr_cluster" "tf-test-cluster" {
         volumes_per_instance = 1
       }
       bid_price = "0.30"
+      autoscaling_policy = <<EOT
+{
+  "Constraints": {
+    "MinCapacity": 1,
+    "MaxCapacity": 2
+  },
+  "Rules": [
+    {
+      "Name": "ScaleOutMemoryPercentage",
+      "Description": "Scale out if YARNMemoryAvailablePercentage is less than 15",
+      "Action": {
+        "SimpleScalingPolicyConfiguration": {
+          "AdjustmentType": "CHANGE_IN_CAPACITY",
+          "ScalingAdjustment": 1,
+          "CoolDown": 300
+        }
+      },
+      "Trigger": {
+        "CloudWatchAlarmDefinition": {
+          "ComparisonOperator": "LESS_THAN",
+          "EvaluationPeriods": 1,
+          "MetricName": "YARNMemoryAvailablePercentage",
+          "Namespace": "AWS/ElasticMapReduce",
+          "Period": 300,
+          "Statistic": "AVERAGE",
+          "Threshold": 15.0,
+          "Unit": "PERCENT"
+        }
+      }
+    }
+  ]
+}
+EOT
     },
     {
       instance_role = "MASTER"

website/docs/r/emr_cluster.html.markdown

@@ -30,6 +30,50 @@ resource "aws_emr_cluster" "emr-test-cluster" {
     instance_profile                  = "${aws_iam_instance_profile.emr_profile.arn}"
   }
 
+  instance_group {
+      instance_role = "CORE"
+      instance_type = "c4.large"
+      instance_count = "1"
+      ebs_config {
+        size = "40"
+        type = "gp2"
+        volumes_per_instance = 1
+      }
+      bid_price = "0.30"
+      autoscaling_policy = <<EOF
+{
+"Constraints": {
+  "MinCapacity": 1,
+  "MaxCapacity": 2
+},
+"Rules": [
+  {
+    "Name": "ScaleOutMemoryPercentage",
+    "Description": "Scale out if YARNMemoryAvailablePercentage is less than 15",
+    "Action": {
+      "SimpleScalingPolicyConfiguration": {
+        "AdjustmentType": "CHANGE_IN_CAPACITY",
+        "ScalingAdjustment": 1,
+        "CoolDown": 300
+      }
+    },
+    "Trigger": {
+      "CloudWatchAlarmDefinition": {
+        "ComparisonOperator": "LESS_THAN",
+        "EvaluationPeriods": 1,
+        "MetricName": "YARNMemoryAvailablePercentage",
+        "Namespace": "AWS/ElasticMapReduce",
+        "Period": 300,
+        "Statistic": "AVERAGE",
+        "Threshold": 15.0,
+        "Unit": "PERCENT"
+      }
+    }
+  }
+]
+}
+EOF
+}
   ebs_root_volume_size     = 100
 
   master_instance_type = "m3.xlarge"
@@ -131,7 +175,7 @@ Attributes for each task instance group in the cluster
 * `name` - (Optional) Friendly name given to the instance group
 * `bid_price` - (Optional) If set, the bid price for each EC2 instance in the instance group, expressed in USD. By setting this attribute, the instance group is being declared as a Spot Instance, and will implicitly create a Spot request. Leave this blank to use On-Demand Instances. `bid_price` can not be set for the `MASTER` instance group, since that group must always be On-Demand
 * `ebs_config` - (Optional) A list of attributes for the EBS volumes attached to each instance in the instance group. Each `ebs_config` defined will result in additional EBS volumes being attached to _each_ instance in the instance group. Defined below
-
+* `autoscaling_policy` - (Optional) The autoscaling policy document. This is a JSON formatted string. See [EMR Auto Scaling](https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-automatic-scaling.html)
 
 ## ebs_config