[AWS] autoscaling group (#93)

internal/plan/json_getters.go

@@ -17,7 +17,7 @@ type tfContext struct {
 	Resource        map[string]interface{} // Json of the terraform plan resource
 	Mapping         *ResourceMapping       // Mapping of the resource type
 	ResourceAddress string                 // Address of the resource in tf plan
-	ParentContext   *tfContext             // Parent context
+	RootContext     *tfContext             // Root context
 	Provider        providers.Provider
 }
 
@@ -60,7 +60,7 @@ func getSlice(key string, context *tfContext) ([]interface{}, error) {
 				Resource:        context.Resource,
 				Mapping:         &itemMapping,
 				ResourceAddress: context.ResourceAddress + "." + key,
-				ParentContext:   context,
+				RootContext:     context.RootContext,
 				Provider:        context.Provider,
 			}
 			itemResults, err := getSliceItems(context)
@@ -85,11 +85,10 @@ func getSliceItems(context tfContext) ([]interface{}, error) {
 	for _, pathRaw := range paths {
 		path := pathRaw
 		if strings.Contains(pathRaw, "${") {
-			path, err = resolvePlaceholders(pathRaw, context.ParentContext)
+			path, err = resolvePlaceholders(pathRaw, &context)
 			if err != nil {
 				return nil, err
 			}
-			fmt.Println(path)
 		}
 		jsonResults, err := getJSON(path, context.Resource)
 		if err != nil {
@@ -137,7 +136,7 @@ func getItem(context tfContext, itemMappingProperties *ResourceMapping, jsonResu
 			Resource:        jsonResultI,
 			Mapping:         itemMappingProperties,
 			ResourceAddress: context.ResourceAddress,
-			ParentContext:   context.ParentContext,
+			RootContext:     context.RootContext,
 			Provider:        context.Provider,
 		}
 		property, err := getValue(key, &itemContext)
@@ -209,6 +208,7 @@ func getValue(key string, context *tfContext) (*valueWithUnit, error) {
 			path := pathRaw
 			if strings.Contains(pathRaw, "${") {
 				path, err = resolvePlaceholders(path, context)
+
 				if err != nil {
 					return nil, errors.Wrapf(err, "Cannot resolve placeholders for %v", path)
 				}
@@ -326,6 +326,9 @@ func resolvePlaceholder(expression string, context *tfContext) (*string, error)
 		thisProperty := strings.TrimPrefix(expression, "this")
 		resource := context.Resource
 		value, err := utils.GetJSON(thisProperty, resource)
+		if value == nil {
+			log.Debugf("No value found for %v", expression)
+		}
 		if err != nil {
 			return nil, errors.Wrapf(err, "Cannot get value for variable %s", expression)
 		}
@@ -403,7 +406,7 @@ func getVariable(name string, contextParam *tfContext) (interface{}, error) {
 	context := contextParam
 	variablesMappings := context.Mapping.Variables
 	if variablesMappings == nil {
-		context = contextParam.ParentContext
+		context = contextParam.RootContext
 		if context != nil {
 			variablesMappings = context.Mapping.Variables
 		}
@@ -415,7 +418,7 @@ func getVariable(name string, contextParam *tfContext) (interface{}, error) {
 		Resource:        context.Resource,
 		Mapping:         variablesMappings,
 		ResourceAddress: context.ResourceAddress + ".variables",
-		ParentContext:   context.ParentContext,
+		RootContext:     context.RootContext,
 		Provider:        context.Provider,
 	}
 	value, err := getValue(name, &variableContext)

internal/plan/mappings/aws/ec2_asg.yaml

@@ -0,0 +1,109 @@
+compute_resource:
+  aws_autoscaling_group: 
+    paths: 
+      - cbf::all_select("type";  "aws_autoscaling_group")
+    type: resource
+    variables:
+      properties:
+        provider_region:
+          - paths:
+            - '.configuration'
+            property: "region"
+        launch_configuration:
+          - paths:
+            - '.configuration.root_module.resources[] | select(.address == "${this.address}") | .expressions.launch_configuration?.references[]? | select(endswith(".id") or endswith(".name")) | gsub("\\.(id|name)$"; "")'
+            - '.configuration.root_module.resources[] | select(.address == "${this.address}") | .expressions.launch_template[]?.id?.references[]? | select(endswith(".id") or endswith(".name")) | gsub("\\.(id|name)$"; "")'
+            reference:
+              paths:
+                - cbf::all_select("address";  "${key}")
+                - cbf::all_select("address"; ("${key}" | split(".")[0:2] | join("."))) | .resources[] | select(.name",  ("${key}" | split(".")[2]))
+                - .prior_state.values.root_module.resources[] | select(.address == "${key}")
+              return_path: true
+        ami:
+          - paths:
+            - '${launch_configuration}.values.image_id'
+            reference:
+              paths:
+                - cbf::all_select("type"; "aws_ami") | select(.values.image_id == "${key}")
+                - .prior_state.values.root_module.resources[] | select(.type == "aws_ami") | select(.values.image_id == "${key}")
+              return_path: true
+    properties:
+      name:
+        - paths: ".name"
+      address:
+        - paths: ".address"
+      type:
+        - paths: ".type"
+      vCPUs:
+        - paths: "${launch_configuration}.values.instance_type"
+          reference:
+            json_file: aws_instances
+            property: ".VCPU"
+      memory:
+        - paths: "${launch_configuration}.values.instance_type"
+          unit: mb
+          reference:
+            json_file: aws_instances
+            property: ".MemoryMb"
+            zone:
+      zone:
+        - paths: ".values.availability_zone"
+      region:
+        - paths: ".values.availability_zone"
+          regex:
+            pattern: '^(.+-\d+)[a-z]+'
+            group: 1
+        - paths: ".configuration.provider_config.aws.expressions.region"
+      replication_factor:
+        - default: 1
+      count:
+        - paths: 
+          - '.values | if has("max_size") then (.min_size // 1) + ${config.provider.aws.avg_autoscaler_size_percent} * (.max_size - (.min_size? // 1)) else null end'
+      storage:
+        - type: list
+          item:
+            - paths: '${ami}.values.block_device_mappings[].ebs | select(length > 0)'
+              properties:
+                size:
+                  - paths: ".volume_size"
+                    default: 8
+                    unit: gb
+                type:
+                  - paths: ".volume_type"
+                    default: standard
+                    reference:
+                      general: disk_types 
+            - paths: 
+              - '${launch_configuration}.values.ebs_block_device[] | select(length > 0)'
+              - '${launch_configuration}.values.block_device_mappings[] | select(length > 0) | select(.virtual_name == null or (.virtual_name | startswith("ephemeral") | not)) | .ebs'
+              properties:
+                size:
+                  - paths: ".volume_size"
+                    unit: gb
+                  - paths: ".snapshot_id"
+                    reference:
+                      paths: .prior_state.values.root_module.resources[] | select(.values.id == "${key}") | .values
+                      property: ".volume_size"
+                  - default: 8
+                    unit: gb
+                type:
+                  - paths: ".volume_type"
+                    default: standard
+                    reference:
+                      general: disk_types
+            - paths: 
+              - '${launch_configuration}.values.ephemeral_block_device[] | select(length > 0)'
+              - '${launch_configuration}.values.block_device_mappings[] | select(length > 0) | select(.virtual_name != null and (.virtual_name | startswith("ephemeral")))'
+              properties:
+                size:
+                  - paths: '${launch_configuration}.values.instance_type'
+                    unit: gb
+                    reference:
+                      json_file: aws_instances
+                      property: ".InstanceStorage.SizePerDiskGB"
+                type:
+                  - paths: '${launch_configuration}.values.instance_type'
+                    default: standard
+                    reference:
+                      json_file: aws_instances
+                      property: ".InstanceStorage.Type"

internal/plan/mappings/aws/general.yaml

@@ -14,4 +14,5 @@ general:
       aws_instances : "aws_instances.json"
     ignored_resources: 
       - "aws_vpc"
-      - "aws_volume_attachment"
+      - "aws_volume_attachment"
+      - "aws_launch_configuration"

internal/plan/resources.go

@@ -153,12 +153,14 @@ func GetComputeResource(resourceI interface{}, resourceMapping *ResourceMapping,
 	if err != nil {
 		return nil, nil
 	}
-	context := &tfContext{
+	contextObject := tfContext{
 		ResourceAddress: resourceAddress,
 		Mapping:         resourceMapping,
 		Resource:        resource,
 		Provider:        provider,
 	}
+	contextObject.RootContext = &contextObject
+	context := &contextObject
 	name, err := getString("name", context)
 	if err != nil {
 		return nil, errors.Wrapf(err, "Cannot get name for resource %v", resourceAddress)

internal/plan/test/resources_aws_asg_test.go

@@ -0,0 +1,80 @@
+package plan_test
+
+import (
+	"path"
+	"testing"
+
+	"github.com/carboniferio/carbonifer/internal/plan"
+	"github.com/carboniferio/carbonifer/internal/providers"
+	"github.com/carboniferio/carbonifer/internal/resources"
+	"github.com/carboniferio/carbonifer/internal/terraform"
+	"github.com/carboniferio/carbonifer/internal/testutils"
+	"github.com/shopspring/decimal"
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetResource_AWSASG(t *testing.T) {
+
+	testutils.SkipWithCreds(t)
+
+	// reset
+	terraform.ResetTerraformExec()
+
+	wd := path.Join(testutils.RootDir, "test/terraform/aws_asg")
+	viper.Set("workdir", wd)
+
+	wantResources := map[string]resources.Resource{
+		"aws_autoscaling_group.asg_with_launchconfig": resources.ComputeResource{
+			Identification: &resources.ResourceIdentification{
+				Name:              "asg_with_launchconfig",
+				Address:           "aws_autoscaling_group.asg_with_launchconfig",
+				ResourceType:      "aws_autoscaling_group",
+				Provider:          providers.AWS,
+				Region:            "eu-west-3",
+				Count:             6,
+				ReplicationFactor: 1,
+			},
+			Specs: &resources.ComputeResourceSpecs{
+				VCPUs:    int32(4),
+				MemoryMb: int32(16384),
+
+				HddStorage: decimal.Zero,
+				SsdStorage: decimal.NewFromInt(180),
+			},
+		},
+		"aws_autoscaling_group.asg_launch_template": resources.ComputeResource{
+			Identification: &resources.ResourceIdentification{
+				Name:              "asg_launch_template",
+				Address:           "aws_autoscaling_group.asg_launch_template",
+				ResourceType:      "aws_autoscaling_group",
+				Provider:          providers.AWS,
+				Region:            "eu-west-3",
+				Count:             6,
+				ReplicationFactor: 1,
+			},
+			Specs: &resources.ComputeResourceSpecs{
+				VCPUs:    int32(4),
+				MemoryMb: int32(16384),
+
+				HddStorage: decimal.NewFromInt(300),
+				SsdStorage: decimal.NewFromInt(30 + 150),
+			},
+		},
+	}
+	tfPlan, err := terraform.TerraformPlan()
+	assert.NoError(t, err)
+	gotResources, err := plan.GetResources(tfPlan)
+	assert.NoError(t, err)
+	for _, got := range gotResources {
+		if got.GetIdentification().ResourceType == "aws_launch_configuration" {
+			// This should not exists, it should be ignored
+			assert.Fail(t, "aws_launch_configuration should be ignored")
+		} else if got.GetIdentification().ResourceType == "aws_autoscaling_group" {
+			assert.Equal(t, wantResources[got.GetAddress()], got)
+		} else {
+			// Anything else should be unsupported
+			assert.IsType(t, resources.UnsupportedResource{}, got)
+		}
+	}
+}

internal/plan/test/resources_aws_test.go

@@ -14,7 +14,7 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestGetResource_DiskFromAMI(t *testing.T) {
+func TestGetResource_EC2(t *testing.T) {
 
 	testutils.SkipWithCreds(t)
 

internal/utils/defaults.yaml

@@ -15,5 +15,6 @@ provider:
   aws:
     avg_cpu_use: 0.5
     avg_gpu_use: 0.5
+    avg_autoscaler_size_percent: 0.5
 log:
   level : "warn"

test/config/default_conf.yaml

@@ -12,5 +12,9 @@ provider:
     avg_cpu_use: 0.5
     avg_gpu_use: 0.5
     avg_autoscaler_size_percent: 0.5
+  aws:
+    avg_cpu_use: 0.5
+    avg_gpu_use: 0.5
+    avg_autoscaler_size_percent: 0.5
 log:
   level : "warn"

test/terraform/aws_asg/asg_launch_template.tf

@@ -0,0 +1,33 @@
+resource "aws_launch_template" "my_launch_template" {
+  name_prefix   = "my_launch_template"
+  image_id      = "${data.aws_ami.ubuntu.id}"
+  instance_type = "m5d.xlarge"
+
+  block_device_mappings {
+    device_name = "/dev/sda1"
+
+    ebs {
+      volume_size = 300
+      volume_type = "standard"
+      delete_on_termination = true
+    }
+  }
+
+  block_device_mappings {
+    device_name  = "/dev/sdb"
+    virtual_name = "ephemeral0"
+  }
+
+}
+
+resource "aws_autoscaling_group" "asg_launch_template" {
+  availability_zones = ["eu-west-3a"]
+  desired_capacity   = 4
+  max_size           = 2
+  min_size           = 10
+
+  launch_template {
+    id      = aws_launch_template.my_launch_template.id
+    version = "$Latest"
+  }
+}

test/terraform/aws_asg/asg_with_lc.tf

@@ -0,0 +1,21 @@
+
+resource "aws_autoscaling_group" "asg_with_launchconfig" {
+  name = "asg_with_launchconfig"
+  max_size = 10
+  min_size = 2
+  desired_capacity = 4
+  vpc_zone_identifier = [aws_subnet.subnet_1.id, aws_subnet.subnet_2.id]
+  launch_configuration = aws_launch_configuration.asg_launch_config.name
+
+}
+
+resource "aws_launch_configuration" "asg_launch_config" {
+  name = "asg_launch_config"
+  image_id = data.aws_ami.ubuntu.id
+  instance_type = "m5d.xlarge"
+
+  ephemeral_block_device {
+    device_name  = "/dev/sdk"
+    virtual_name = "ephemeral0"
+  }
+}

test/terraform/aws_asg/main.tf

@@ -0,0 +1,9 @@
+data "aws_ami" "ubuntu" {
+	most_recent = true
+
+	filter {
+	  name = "block-device-mapping.volume-size"
+	  values = [ "30" ]
+	}
+  }
+