From ae97944e9a19903483efa649b954d53223c97ef4 Mon Sep 17 00:00:00 2001 From: Olivier Bierlaire Date: Tue, 12 Sep 2023 16:42:19 +0200 Subject: [PATCH] Add missing AWS/EC2 ephemeral storages (#91) --- internal/plan/json_getters.go | 12 ++++++-- internal/plan/mappings/aws/ec2_ebs.yaml | 2 +- internal/plan/mappings/aws/ec2_instance.yaml | 24 +++++++++++++--- internal/plan/mappings/aws/rds_instance.yaml | 4 +-- internal/plan/mappings/gcp/compute.yaml | 12 ++++---- internal/plan/mappings/gcp/compute_group.yaml | 6 ++-- internal/plan/mappings/gcp/disk.yaml | 2 +- internal/plan/mappings/gcp/gke.yaml | 4 +-- internal/plan/mappings/gcp/sql_database.yaml | 4 +-- internal/plan/resolver.go | 28 +++++++++++++++---- internal/plan/test/resources_aws_test.go | 6 ++-- test/terraform/aws_ec2/main.tf | 12 +++++++- 12 files changed, 83 insertions(+), 33 deletions(-) diff --git a/internal/plan/json_getters.go b/internal/plan/json_getters.go index c01f2db..cbc30c6 100644 --- a/internal/plan/json_getters.go +++ b/internal/plan/json_getters.go @@ -89,6 +89,7 @@ func getSliceItems(context tfContext) ([]interface{}, error) { if err != nil { return nil, err } + fmt.Println(path) } jsonResults, err := getJSON(path, context.Resource) if err != nil { @@ -136,7 +137,7 @@ func getItem(context tfContext, itemMappingProperties *ResourceMapping, jsonResu Resource: jsonResultI, Mapping: itemMappingProperties, ResourceAddress: context.ResourceAddress, - ParentContext: &context, + ParentContext: context.ParentContext, Provider: context.Provider, } property, err := getValue(key, &itemContext) @@ -398,8 +399,15 @@ func getDefaultValue(key string, context *tfContext) (*valueWithUnit, error) { } -func getVariable(name string, context *tfContext) (interface{}, error) { +func getVariable(name string, contextParam *tfContext) (interface{}, error) { + context := contextParam variablesMappings := context.Mapping.Variables + if variablesMappings == nil { + context = contextParam.ParentContext + if context != nil { + variablesMappings = context.Mapping.Variables + } + } if variablesMappings == nil { return nil, nil } diff --git a/internal/plan/mappings/aws/ec2_ebs.yaml b/internal/plan/mappings/aws/ec2_ebs.yaml index f7b36d8..3ce096a 100644 --- a/internal/plan/mappings/aws/ec2_ebs.yaml +++ b/internal/plan/mappings/aws/ec2_ebs.yaml @@ -30,7 +30,7 @@ compute_resource: - paths: ".snapshot_id" reference: paths: .prior_state.values.root_module.resources[] | select(.values.id == "${key}") | .values - property: "volume_size" + property: ".volume_size" - default: 8 unit: gb type: diff --git a/internal/plan/mappings/aws/ec2_instance.yaml b/internal/plan/mappings/aws/ec2_instance.yaml index 1fd0e29..cc7179d 100644 --- a/internal/plan/mappings/aws/ec2_instance.yaml +++ b/internal/plan/mappings/aws/ec2_instance.yaml @@ -14,6 +14,8 @@ compute_resource: - paths: - '.configuration' property: "region" + instance_type: + - paths: '.values.instance_type' properties: name: - paths: ".name" @@ -25,13 +27,13 @@ compute_resource: - paths: ".values.instance_type" reference: json_file: aws_instances - property: "VCPU" + property: ".VCPU" memory: - paths: ".values.instance_type" unit: mb reference: json_file: aws_instances - property: "MemoryMb" + property: ".MemoryMb" zone: zone: - paths: ".values.availability_zone" @@ -65,11 +67,25 @@ compute_resource: - paths: ".snapshot_id" reference: paths: .prior_state.values.root_module.resources[] | select(.values.id == "${key}") | .values - property: "volume_size" + property: ".volume_size" - default: 8 unit: gb type: - paths: ".volume_type" default: standard reference: - general: disk_types \ No newline at end of file + general: disk_types + - paths: '.values.ephemeral_block_device[] | select(length > 0)' + properties: + size: + - paths: '"${instance_type}"' + unit: gb + reference: + json_file: aws_instances + property: ".InstanceStorage.SizePerDiskGB" + type: + - paths: '"${instance_type}"' + default: standard + reference: + json_file: aws_instances + property: ".InstanceStorage.Type" \ No newline at end of file diff --git a/internal/plan/mappings/aws/rds_instance.yaml b/internal/plan/mappings/aws/rds_instance.yaml index 6bf2215..805adab 100644 --- a/internal/plan/mappings/aws/rds_instance.yaml +++ b/internal/plan/mappings/aws/rds_instance.yaml @@ -38,7 +38,7 @@ compute_resource: group: 1 reference: json_file: aws_instances - property: "VCPU" + property: ".VCPU" memory: - paths: ".values.instance_class" unit: mb @@ -47,7 +47,7 @@ compute_resource: group: 1 reference: json_file: aws_instances - property: "MemoryMb" + property: ".MemoryMb" storage: - type: list item: diff --git a/internal/plan/mappings/gcp/compute.yaml b/internal/plan/mappings/gcp/compute.yaml index 0df4a8c..ecff746 100644 --- a/internal/plan/mappings/gcp/compute.yaml +++ b/internal/plan/mappings/gcp/compute.yaml @@ -13,7 +13,7 @@ compute_resource: - paths: ".values.machine_type" reference: json_file: gcp_machines_types - property: "vcpus" + property: ".vcpus" - paths: ".values.machine_type" regex: pattern: ".*custom-([0-9]+)-.*" @@ -24,7 +24,7 @@ compute_resource: unit: mb reference: json_file: gcp_machines_types - property: "memoryMb" + property: ".memoryMb" - paths: ".values.machine_type" unit: mb regex: @@ -73,7 +73,7 @@ compute_resource: unit: gb reference: paths: '.prior_state.values.root_module.resources[] | select(.values.self_link == "${key}") | .values' - property: "disk_size_gb" + property: ".disk_size_gb" type: - paths: - ".type" @@ -115,13 +115,13 @@ compute_resource: - paths: "${template_config}.values.machine_type" reference: json_file: gcp_machines_types - property: "vcpus" + property: ".vcpus" memory: - paths: "${template_config}.values.machine_type" unit: mb reference: json_file: gcp_machines_types - property: "memoryMb" + property: ".memoryMb" zone: - paths: ".values.zone" region: @@ -156,7 +156,7 @@ compute_resource: unit: gb reference: paths: .prior_state.values.root_module.resources[] | select(.values.self_link == "${key}") | .values - property: "disk_size_gb" + property: ".disk_size_gb" - default: 10 type: - paths: ".disk_type" diff --git a/internal/plan/mappings/gcp/compute_group.yaml b/internal/plan/mappings/gcp/compute_group.yaml index b43b76c..aa55c85 100644 --- a/internal/plan/mappings/gcp/compute_group.yaml +++ b/internal/plan/mappings/gcp/compute_group.yaml @@ -34,7 +34,7 @@ compute_resource: - paths: "${template_config}.values.machine_type" reference: json_file: gcp_machines_types - property: "vcpus" + property: ".vcpus" - paths: "${template_config}.values.machine_type" regex: pattern: ".*custom-([0-9]+)-.*" @@ -45,7 +45,7 @@ compute_resource: unit: mb reference: json_file: gcp_machines_types - property: "memoryMb" + property: ".memoryMb" - paths: "${template_config}.values.machine_type" unit: mb regex: @@ -93,7 +93,7 @@ compute_resource: unit: gb reference: paths: .prior_state.values.root_module.resources[] | select(.values.self_link == "${key}") | .values - property: "disk_size_gb" + property: ".disk_size_gb" - default: 10 type: - paths: ".disk_type" diff --git a/internal/plan/mappings/gcp/disk.yaml b/internal/plan/mappings/gcp/disk.yaml index 87fb7b2..866a1d1 100644 --- a/internal/plan/mappings/gcp/disk.yaml +++ b/internal/plan/mappings/gcp/disk.yaml @@ -38,7 +38,7 @@ compute_resource: unit: gb reference: paths: .prior_state.values.root_module.resources[] | select(.values.self_link == "${key}") | .values - property: "disk_size_gb" + property: ".disk_size_gb" - default: 10 unit: gb type: diff --git a/internal/plan/mappings/gcp/gke.yaml b/internal/plan/mappings/gcp/gke.yaml index b59e83e..c3b795d 100644 --- a/internal/plan/mappings/gcp/gke.yaml +++ b/internal/plan/mappings/gcp/gke.yaml @@ -33,7 +33,7 @@ compute_resource: - "${node_pool}.node_config[].machine_type" reference: json_file: gcp_machines_types - property: "vcpus" + property: ".vcpus" - paths: - ".values.node_config[].machine_type" - "${node_pool}.node_config[].machine_type" @@ -52,7 +52,7 @@ compute_resource: unit: mb reference: json_file: gcp_machines_types - property: "memoryMb" + property: ".memoryMb" - paths: - ".values.node_config[].machine_type" - "${node_pool}.node_config[].machine_type" diff --git a/internal/plan/mappings/gcp/sql_database.yaml b/internal/plan/mappings/gcp/sql_database.yaml index 453e18e..fa8b0f4 100644 --- a/internal/plan/mappings/gcp/sql_database.yaml +++ b/internal/plan/mappings/gcp/sql_database.yaml @@ -13,13 +13,13 @@ compute_resource: - paths: ".values.settings[0].tier" reference: json_file: gcp_sql_tiers - property: "vcpus" + property: ".vcpus" memory: - paths: ".values.settings[0].tier" unit: mb reference: json_file: gcp_sql_tiers - property: "memoryMb" + property: ".memoryMb" zone: - paths: ".values.zone" region: diff --git a/internal/plan/resolver.go b/internal/plan/resolver.go index 39ac80e..6181148 100644 --- a/internal/plan/resolver.go +++ b/internal/plan/resolver.go @@ -6,6 +6,7 @@ import ( "regexp" "github.com/carboniferio/carbonifer/internal/data" + "github.com/carboniferio/carbonifer/internal/utils" "github.com/pkg/errors" "github.com/shopspring/decimal" log "github.com/sirupsen/logrus" @@ -44,12 +45,21 @@ func resolveReference(key string, reference *Reference, context *tfContext) (int log.Debugf("Cannot find key %v in file %v", key, reference.JSONFile) return nil, nil } + var value interface{} property := reference.Property if property != "" { - value, ok = item.(map[string]interface{})[reference.Property] - if !ok { - log.Fatalf("Cannot find property %v in file %v", reference.Property, reference.JSONFile) + valueArray, err := utils.GetJSON(reference.Property, item) + if err != nil { + return nil, errors.Wrapf(err, "Cannot find property %v in file %v", reference.Property, reference.JSONFile) + } + if len(valueArray) != 0 { + if valueArray[0] == nil { + return nil, fmt.Errorf("Cannot find property %v in file %v", reference.Property, reference.JSONFile) + } + value = valueArray[0] + } else { + return nil, fmt.Errorf("Cannot find property %v in file %v", reference.Property, reference.JSONFile) } } return value, nil @@ -82,10 +92,16 @@ func resolveReference(key string, reference *Reference, context *tfContext) (int } for _, referencedItem := range referencedItems { if reference.Property != "" { - value := referencedItem.(map[string]interface{})[reference.Property] - if value != nil { - return value, nil + value, err := utils.GetJSON(reference.Property, referencedItem) + if err != nil { + return nil, errors.Wrapf(err, "Cannot find property %v in path %v", reference.Property, path) + } + if len(value) != 0 { + if value[0] != nil { + return value[0], nil + } } + } else if reference.ReturnPath { return path, nil } else { diff --git a/internal/plan/test/resources_aws_test.go b/internal/plan/test/resources_aws_test.go index 6a15e4b..a5a4deb 100644 --- a/internal/plan/test/resources_aws_test.go +++ b/internal/plan/test/resources_aws_test.go @@ -36,11 +36,11 @@ func TestGetResource_DiskFromAMI(t *testing.T) { ReplicationFactor: 1, }, Specs: &resources.ComputeResourceSpecs{ - VCPUs: int32(2), - MemoryMb: int32(8192), + VCPUs: int32(4), + MemoryMb: int32(16384), HddStorage: decimal.NewFromInt(80), - SsdStorage: decimal.NewFromInt(30), + SsdStorage: decimal.NewFromInt(330), }, }, "aws_ebs_volume.ebs_volume": resources.ComputeResource{ diff --git a/test/terraform/aws_ec2/main.tf b/test/terraform/aws_ec2/main.tf index a6b1e86..143093e 100644 --- a/test/terraform/aws_ec2/main.tf +++ b/test/terraform/aws_ec2/main.tf @@ -36,7 +36,7 @@ resource "aws_ebs_volume" "ebs_volume" { resource "aws_instance" "foo" { ami = data.aws_ami.ubuntu.id - instance_type = "m4.large" + instance_type = "m5d.xlarge" network_interface { network_interface_id = aws_network_interface.foo.id @@ -53,6 +53,16 @@ resource "aws_instance" "foo" { device_name = "/dev/sdj" snapshot_id = data.aws_ebs_snapshot.ebs_snapshot.id } + + ephemeral_block_device { + device_name = "/dev/sdk" + virtual_name = "ephemeral0" + } + + ephemeral_block_device { + device_name = "/dev/sdl" + virtual_name = "ephemeral1" + } } resource "aws_volume_attachment" "ebs_att" {