diff --git a/internal/plan/json_getters.go b/internal/plan/json_getters.go index cbc30c6..2fb1246 100644 --- a/internal/plan/json_getters.go +++ b/internal/plan/json_getters.go @@ -17,7 +17,7 @@ type tfContext struct { Resource map[string]interface{} // Json of the terraform plan resource Mapping *ResourceMapping // Mapping of the resource type ResourceAddress string // Address of the resource in tf plan - ParentContext *tfContext // Parent context + RootContext *tfContext // Root context Provider providers.Provider } @@ -60,7 +60,7 @@ func getSlice(key string, context *tfContext) ([]interface{}, error) { Resource: context.Resource, Mapping: &itemMapping, ResourceAddress: context.ResourceAddress + "." + key, - ParentContext: context, + RootContext: context.RootContext, Provider: context.Provider, } itemResults, err := getSliceItems(context) @@ -85,11 +85,10 @@ func getSliceItems(context tfContext) ([]interface{}, error) { for _, pathRaw := range paths { path := pathRaw if strings.Contains(pathRaw, "${") { - path, err = resolvePlaceholders(pathRaw, context.ParentContext) + path, err = resolvePlaceholders(pathRaw, &context) if err != nil { return nil, err } - fmt.Println(path) } jsonResults, err := getJSON(path, context.Resource) if err != nil { @@ -137,7 +136,7 @@ func getItem(context tfContext, itemMappingProperties *ResourceMapping, jsonResu Resource: jsonResultI, Mapping: itemMappingProperties, ResourceAddress: context.ResourceAddress, - ParentContext: context.ParentContext, + RootContext: context.RootContext, Provider: context.Provider, } property, err := getValue(key, &itemContext) @@ -209,6 +208,7 @@ func getValue(key string, context *tfContext) (*valueWithUnit, error) { path := pathRaw if strings.Contains(pathRaw, "${") { path, err = resolvePlaceholders(path, context) + if err != nil { return nil, errors.Wrapf(err, "Cannot resolve placeholders for %v", path) } @@ -326,6 +326,9 @@ func resolvePlaceholder(expression string, context *tfContext) (*string, error) thisProperty := strings.TrimPrefix(expression, "this") resource := context.Resource value, err := utils.GetJSON(thisProperty, resource) + if value == nil { + log.Debugf("No value found for %v", expression) + } if err != nil { return nil, errors.Wrapf(err, "Cannot get value for variable %s", expression) } @@ -403,7 +406,7 @@ func getVariable(name string, contextParam *tfContext) (interface{}, error) { context := contextParam variablesMappings := context.Mapping.Variables if variablesMappings == nil { - context = contextParam.ParentContext + context = contextParam.RootContext if context != nil { variablesMappings = context.Mapping.Variables } @@ -415,7 +418,7 @@ func getVariable(name string, contextParam *tfContext) (interface{}, error) { Resource: context.Resource, Mapping: variablesMappings, ResourceAddress: context.ResourceAddress + ".variables", - ParentContext: context.ParentContext, + RootContext: context.RootContext, Provider: context.Provider, } value, err := getValue(name, &variableContext) diff --git a/internal/plan/mappings/aws/ec2_asg.yaml b/internal/plan/mappings/aws/ec2_asg.yaml new file mode 100644 index 0000000..b0ea206 --- /dev/null +++ b/internal/plan/mappings/aws/ec2_asg.yaml @@ -0,0 +1,109 @@ +compute_resource: + aws_autoscaling_group: + paths: + - cbf::all_select("type"; "aws_autoscaling_group") + type: resource + variables: + properties: + provider_region: + - paths: + - '.configuration' + property: "region" + launch_configuration: + - paths: + - '.configuration.root_module.resources[] | select(.address == "${this.address}") | .expressions.launch_configuration?.references[]? | select(endswith(".id") or endswith(".name")) | gsub("\\.(id|name)$"; "")' + - '.configuration.root_module.resources[] | select(.address == "${this.address}") | .expressions.launch_template[]?.id?.references[]? | select(endswith(".id") or endswith(".name")) | gsub("\\.(id|name)$"; "")' + reference: + paths: + - cbf::all_select("address"; "${key}") + - cbf::all_select("address"; ("${key}" | split(".")[0:2] | join("."))) | .resources[] | select(.name", ("${key}" | split(".")[2])) + - .prior_state.values.root_module.resources[] | select(.address == "${key}") + return_path: true + ami: + - paths: + - '${launch_configuration}.values.image_id' + reference: + paths: + - cbf::all_select("type"; "aws_ami") | select(.values.image_id == "${key}") + - .prior_state.values.root_module.resources[] | select(.type == "aws_ami") | select(.values.image_id == "${key}") + return_path: true + properties: + name: + - paths: ".name" + address: + - paths: ".address" + type: + - paths: ".type" + vCPUs: + - paths: "${launch_configuration}.values.instance_type" + reference: + json_file: aws_instances + property: ".VCPU" + memory: + - paths: "${launch_configuration}.values.instance_type" + unit: mb + reference: + json_file: aws_instances + property: ".MemoryMb" + zone: + zone: + - paths: ".values.availability_zone" + region: + - paths: ".values.availability_zone" + regex: + pattern: '^(.+-\d+)[a-z]+' + group: 1 + - paths: ".configuration.provider_config.aws.expressions.region" + replication_factor: + - default: 1 + count: + - paths: + - '.values | if has("max_size") then (.min_size // 1) + ${config.provider.aws.avg_autoscaler_size_percent} * (.max_size - (.min_size? // 1)) else null end' + storage: + - type: list + item: + - paths: '${ami}.values.block_device_mappings[].ebs | select(length > 0)' + properties: + size: + - paths: ".volume_size" + default: 8 + unit: gb + type: + - paths: ".volume_type" + default: standard + reference: + general: disk_types + - paths: + - '${launch_configuration}.values.ebs_block_device[] | select(length > 0)' + - '${launch_configuration}.values.block_device_mappings[] | select(length > 0) | select(.virtual_name == null or (.virtual_name | startswith("ephemeral") | not)) | .ebs' + properties: + size: + - paths: ".volume_size" + unit: gb + - paths: ".snapshot_id" + reference: + paths: .prior_state.values.root_module.resources[] | select(.values.id == "${key}") | .values + property: ".volume_size" + - default: 8 + unit: gb + type: + - paths: ".volume_type" + default: standard + reference: + general: disk_types + - paths: + - '${launch_configuration}.values.ephemeral_block_device[] | select(length > 0)' + - '${launch_configuration}.values.block_device_mappings[] | select(length > 0) | select(.virtual_name != null and (.virtual_name | startswith("ephemeral")))' + properties: + size: + - paths: '${launch_configuration}.values.instance_type' + unit: gb + reference: + json_file: aws_instances + property: ".InstanceStorage.SizePerDiskGB" + type: + - paths: '${launch_configuration}.values.instance_type' + default: standard + reference: + json_file: aws_instances + property: ".InstanceStorage.Type" \ No newline at end of file diff --git a/internal/plan/mappings/aws/general.yaml b/internal/plan/mappings/aws/general.yaml index 64cba7c..cb8f127 100644 --- a/internal/plan/mappings/aws/general.yaml +++ b/internal/plan/mappings/aws/general.yaml @@ -14,4 +14,5 @@ general: aws_instances : "aws_instances.json" ignored_resources: - "aws_vpc" - - "aws_volume_attachment" \ No newline at end of file + - "aws_volume_attachment" + - "aws_launch_configuration" \ No newline at end of file diff --git a/internal/plan/resources.go b/internal/plan/resources.go index 2a23fb5..a2b575f 100644 --- a/internal/plan/resources.go +++ b/internal/plan/resources.go @@ -153,12 +153,14 @@ func GetComputeResource(resourceI interface{}, resourceMapping *ResourceMapping, if err != nil { return nil, nil } - context := &tfContext{ + contextObject := tfContext{ ResourceAddress: resourceAddress, Mapping: resourceMapping, Resource: resource, Provider: provider, } + contextObject.RootContext = &contextObject + context := &contextObject name, err := getString("name", context) if err != nil { return nil, errors.Wrapf(err, "Cannot get name for resource %v", resourceAddress) diff --git a/internal/plan/test/resources_aws_asg_test.go b/internal/plan/test/resources_aws_asg_test.go new file mode 100644 index 0000000..b27f86a --- /dev/null +++ b/internal/plan/test/resources_aws_asg_test.go @@ -0,0 +1,80 @@ +package plan_test + +import ( + "path" + "testing" + + "github.com/carboniferio/carbonifer/internal/plan" + "github.com/carboniferio/carbonifer/internal/providers" + "github.com/carboniferio/carbonifer/internal/resources" + "github.com/carboniferio/carbonifer/internal/terraform" + "github.com/carboniferio/carbonifer/internal/testutils" + "github.com/shopspring/decimal" + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" +) + +func TestGetResource_AWSASG(t *testing.T) { + + testutils.SkipWithCreds(t) + + // reset + terraform.ResetTerraformExec() + + wd := path.Join(testutils.RootDir, "test/terraform/aws_asg") + viper.Set("workdir", wd) + + wantResources := map[string]resources.Resource{ + "aws_autoscaling_group.asg_with_launchconfig": resources.ComputeResource{ + Identification: &resources.ResourceIdentification{ + Name: "asg_with_launchconfig", + Address: "aws_autoscaling_group.asg_with_launchconfig", + ResourceType: "aws_autoscaling_group", + Provider: providers.AWS, + Region: "eu-west-3", + Count: 6, + ReplicationFactor: 1, + }, + Specs: &resources.ComputeResourceSpecs{ + VCPUs: int32(4), + MemoryMb: int32(16384), + + HddStorage: decimal.Zero, + SsdStorage: decimal.NewFromInt(180), + }, + }, + "aws_autoscaling_group.asg_launch_template": resources.ComputeResource{ + Identification: &resources.ResourceIdentification{ + Name: "asg_launch_template", + Address: "aws_autoscaling_group.asg_launch_template", + ResourceType: "aws_autoscaling_group", + Provider: providers.AWS, + Region: "eu-west-3", + Count: 6, + ReplicationFactor: 1, + }, + Specs: &resources.ComputeResourceSpecs{ + VCPUs: int32(4), + MemoryMb: int32(16384), + + HddStorage: decimal.NewFromInt(300), + SsdStorage: decimal.NewFromInt(30 + 150), + }, + }, + } + tfPlan, err := terraform.TerraformPlan() + assert.NoError(t, err) + gotResources, err := plan.GetResources(tfPlan) + assert.NoError(t, err) + for _, got := range gotResources { + if got.GetIdentification().ResourceType == "aws_launch_configuration" { + // This should not exists, it should be ignored + assert.Fail(t, "aws_launch_configuration should be ignored") + } else if got.GetIdentification().ResourceType == "aws_autoscaling_group" { + assert.Equal(t, wantResources[got.GetAddress()], got) + } else { + // Anything else should be unsupported + assert.IsType(t, resources.UnsupportedResource{}, got) + } + } +} diff --git a/internal/plan/test/resources_aws_test.go b/internal/plan/test/resources_aws_test.go index a5a4deb..56a5d99 100644 --- a/internal/plan/test/resources_aws_test.go +++ b/internal/plan/test/resources_aws_test.go @@ -14,7 +14,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestGetResource_DiskFromAMI(t *testing.T) { +func TestGetResource_EC2(t *testing.T) { testutils.SkipWithCreds(t) diff --git a/internal/utils/defaults.yaml b/internal/utils/defaults.yaml index 636268a..f9ffa06 100644 --- a/internal/utils/defaults.yaml +++ b/internal/utils/defaults.yaml @@ -15,5 +15,6 @@ provider: aws: avg_cpu_use: 0.5 avg_gpu_use: 0.5 + avg_autoscaler_size_percent: 0.5 log: level : "warn" diff --git a/test/config/default_conf.yaml b/test/config/default_conf.yaml index 4f1dfd8..95ad694 100644 --- a/test/config/default_conf.yaml +++ b/test/config/default_conf.yaml @@ -12,5 +12,9 @@ provider: avg_cpu_use: 0.5 avg_gpu_use: 0.5 avg_autoscaler_size_percent: 0.5 + aws: + avg_cpu_use: 0.5 + avg_gpu_use: 0.5 + avg_autoscaler_size_percent: 0.5 log: level : "warn" \ No newline at end of file diff --git a/test/terraform/aws_asg/asg_launch_template.tf b/test/terraform/aws_asg/asg_launch_template.tf new file mode 100644 index 0000000..ef811d4 --- /dev/null +++ b/test/terraform/aws_asg/asg_launch_template.tf @@ -0,0 +1,33 @@ +resource "aws_launch_template" "my_launch_template" { + name_prefix = "my_launch_template" + image_id = "${data.aws_ami.ubuntu.id}" + instance_type = "m5d.xlarge" + + block_device_mappings { + device_name = "/dev/sda1" + + ebs { + volume_size = 300 + volume_type = "standard" + delete_on_termination = true + } + } + + block_device_mappings { + device_name = "/dev/sdb" + virtual_name = "ephemeral0" + } + +} + +resource "aws_autoscaling_group" "asg_launch_template" { + availability_zones = ["eu-west-3a"] + desired_capacity = 4 + max_size = 2 + min_size = 10 + + launch_template { + id = aws_launch_template.my_launch_template.id + version = "$Latest" + } +} \ No newline at end of file diff --git a/test/terraform/aws_asg/asg_with_lc.tf b/test/terraform/aws_asg/asg_with_lc.tf new file mode 100644 index 0000000..676af1e --- /dev/null +++ b/test/terraform/aws_asg/asg_with_lc.tf @@ -0,0 +1,21 @@ + +resource "aws_autoscaling_group" "asg_with_launchconfig" { + name = "asg_with_launchconfig" + max_size = 10 + min_size = 2 + desired_capacity = 4 + vpc_zone_identifier = [aws_subnet.subnet_1.id, aws_subnet.subnet_2.id] + launch_configuration = aws_launch_configuration.asg_launch_config.name + +} + +resource "aws_launch_configuration" "asg_launch_config" { + name = "asg_launch_config" + image_id = data.aws_ami.ubuntu.id + instance_type = "m5d.xlarge" + + ephemeral_block_device { + device_name = "/dev/sdk" + virtual_name = "ephemeral0" + } +} \ No newline at end of file diff --git a/test/terraform/aws_asg/main.tf b/test/terraform/aws_asg/main.tf new file mode 100644 index 0000000..b1d1620 --- /dev/null +++ b/test/terraform/aws_asg/main.tf @@ -0,0 +1,9 @@ +data "aws_ami" "ubuntu" { + most_recent = true + + filter { + name = "block-device-mapping.volume-size" + values = [ "30" ] + } + } + \ No newline at end of file diff --git a/test/terraform/aws_asg/network.tf b/test/terraform/aws_asg/network.tf new file mode 100644 index 0000000..23106a3 --- /dev/null +++ b/test/terraform/aws_asg/network.tf @@ -0,0 +1,43 @@ +resource "aws_vpc" "my_vpc" { + cidr_block = "172.16.0.0/16" + + tags = { + Name = "tf-example" + } +} + +resource "aws_subnet" "subnet_1" { + vpc_id = aws_vpc.my_vpc.id + cidr_block = "172.16.10.0/24" + availability_zone = "us-west-2a" + tags = { + Name = "tf-example" + } +} + +resource "aws_network_interface" "network_interface_1" { + subnet_id = aws_subnet.subnet_1.id + private_ips = ["172.16.10.100"] + + tags = { + Name = "primary_network_interface" + } +} + +resource "aws_subnet" "subnet_2" { + vpc_id = aws_vpc.my_vpc.id + cidr_block = "172.16.20.0/24" + availability_zone = "us-west-2a" + tags = { + Name = "tf-example" + } +} + +resource "aws_network_interface" "network_interface_2" { + subnet_id = aws_subnet.subnet_2.id + private_ips = ["172.16.20.100"] + + tags = { + Name = "primary_network_interface" + } +} \ No newline at end of file diff --git a/test/terraform/aws_asg/provider.tf b/test/terraform/aws_asg/provider.tf new file mode 100644 index 0000000..5b8c3cb --- /dev/null +++ b/test/terraform/aws_asg/provider.tf @@ -0,0 +1,4 @@ +provider "aws" { + region = "eu-west-3" +} + diff --git a/test/terraform/planRaw/plan.tfplan b/test/terraform/planRaw/plan.tfplan index b57ab48..9fffff4 100644 Binary files a/test/terraform/planRaw/plan.tfplan and b/test/terraform/planRaw/plan.tfplan differ