Estimate resources in child modules (#82)

internal/output/text.go

@@ -17,7 +17,7 @@ func GenerateReportText(report estimation.EstimationReport) string {
 	tableString.WriteString("\n  Average estimation of CO2 emissions per instance: \n\n")
 
 	table := tablewriter.NewWriter(tableString)
-	table.SetHeader([]string{"resource type", "name", "count", "emissions per instance"})
+	table.SetHeader([]string{"resource type", "address", "count", "emissions per instance"})
 
 	// Default sort
 	estimations := report.Resources
@@ -26,7 +26,7 @@ func GenerateReportText(report estimation.EstimationReport) string {
 	for _, resource := range report.Resources {
 		table.Append([]string{
 			resource.Resource.GetIdentification().ResourceType,
-			resource.Resource.GetIdentification().Name,
+			resource.Resource.GetAddress(),
 			fmt.Sprintf("%v", resource.Count),
 			fmt.Sprintf(" %v %v", resource.CarbonEmissions.StringFixed(4), report.Info.UnitCarbonEmissionsTime),
 		})

internal/plan/json.go

@@ -0,0 +1,34 @@
+package plan
+
+import (
+	"strings"
+
+	"github.com/carboniferio/carbonifer/internal/utils"
+)
+
+const allResourcesQuery = ".planned_values | .. | objects | select(has(\"resources\")) | .resources[]"
+
+func getJSON(query string, json interface{}) ([]interface{}, error) {
+
+	if strings.HasPrefix(query, "select(") {
+		results, err := utils.GetJSON(allResourcesQuery+" | "+query, *TfPlan)
+		if len(results) > 0 && err == nil {
+			return results, nil
+		}
+		return nil, err
+	}
+
+	if strings.HasPrefix(query, ".configuration") || strings.HasPrefix(query, ".prior_state") || strings.HasPrefix(query, ".planned_values") {
+		results, err := utils.GetJSON(query, *TfPlan)
+		if len(results) > 0 && err == nil {
+			return results, nil
+		}
+		return nil, err
+	}
+
+	results, err := utils.GetJSON(query, json)
+	if len(results) > 0 && err == nil {
+		return results, nil
+	}
+	return nil, err
+}

internal/plan/json_getters.go

@@ -90,17 +90,10 @@ func getSliceItems(context tfContext) ([]interface{}, error) {
 				return nil, err
 			}
 		}
-		jsonResults, err := utils.GetJSON(path, context.Resource)
+		jsonResults, err := getJSON(path, context.Resource)
 		if err != nil {
 			return nil, errors.Wrapf(err, "Cannot get item: %v", path)
 		}
-		// if no result, try to get it from the whole plan
-		if len(jsonResults) == 0 && TfPlan != nil {
-			jsonResults, err = utils.GetJSON(path, *TfPlan)
-			if err != nil {
-				return nil, errors.Wrapf(err, "Cannot get item in full plan: %v", path)
-			}
-		}
 		for _, jsonResultsI := range jsonResults {
 			switch jsonResults := jsonResultsI.(type) {
 			case map[string]interface{}:
@@ -214,24 +207,15 @@ func getValue(key string, context *tfContext) (*valueWithUnit, error) {
 			}
 			path := pathRaw
 			if strings.Contains(pathRaw, "${") {
-				fmt.Println("pathRaw: ", pathRaw)
 				path, err = resolvePlaceholders(path, context)
 				if err != nil {
 					return nil, errors.Wrapf(err, "Cannot resolve placeholders for %v", path)
 				}
-				fmt.Println("path: ", path)
 			}
-			valueFounds, err := utils.GetJSON(path, context.Resource)
+			valueFounds, err := getJSON(path, context.Resource)
 			if err != nil {
 				return nil, errors.Wrapf(err, "Cannot get value for %v", path)
 			}
-			if len(valueFounds) == 0 && TfPlan != nil {
-				// Try to resolve it against the whole plan
-				valueFounds, err = utils.GetJSON(path, *TfPlan)
-				if err != nil {
-					return nil, errors.Wrapf(err, "Cannot get value in the whole plan for %v", path)
-				}
-			}
 			if len(valueFounds) > 0 {
 				if len(valueFounds) > 1 {
 					return nil, errors.Errorf("Found more than one value for property %v of resource type %v", key, context.ResourceAddress)

internal/plan/mappings/aws/ec2_ebs.yaml

@@ -1,7 +1,7 @@
 compute_resource:
   aws_ebs_volume:
     paths:
-      - .planned_values.root_module.resources[] | select(.type == "aws_ebs_volume")
+      - select(.type == "aws_ebs_volume")
     type: resource
     properties:
       name:

internal/plan/mappings/aws/ec2_instance.yaml

@@ -1,13 +1,13 @@
 compute_resource:
   aws_instance: 
     paths: 
-      - .planned_values.root_module.resources[] | select(.type == "aws_instance")
+      - select(.type == "aws_instance")
     type: resource
     variables:
       properties:
         ami:
           - paths:
-            - '.prior_state.values.root_module.resources[] | select(.values.image_id == "${this.values.ami}")'
+            - select(.values.image_id == "${this.values.ami}")
             reference:
               return_path: true
         provider_region:

internal/plan/mappings/aws/rds_instance.yaml

@@ -1,7 +1,7 @@
 compute_resource:
   aws_db_instance: 
     paths: 
-      - .planned_values.root_module.resources[] | select(.type == "aws_db_instance")
+      - select(.type == "aws_db_instance")
     type: resource
     variables:
       properties:
@@ -10,8 +10,8 @@ compute_resource:
             - '.configuration.root_module.resources[] | select(.address == "${this.address}") | .expressions.replicate_source_db?.references[]? | select(endswith("id")) | gsub("\\.id$"; "")'
             reference:
               paths:
-                - .planned_values.root_module.resources[] | select(.address == "${key}")
-                - .planned_values.root_module.child_modules[] | select(.address == ("${key}" | split(".")[0:2] | join("."))) | .resources[] | select(.name == ("${key}" | split(".")[2]))
+                - select(.address == "${key}")
+                - select(.address == ("${key}" | split(".")[0:2] | join("."))) | .resources[] | select(.name == ("${key}" | split(".")[2]))
                 - .prior_state.values.root_module.resources[] | select(.address == "${key}")
               return_path: true
     properties:

internal/plan/mappings/gcp/compute.yaml

@@ -1,6 +1,6 @@
 compute_resource:
   google_compute_instance:
-    paths: .planned_values.root_module.resources[] | select(.type == "google_compute_instance")
+    paths: 'select(.type == "google_compute_instance")'
     type: resource
     properties:
       name:
@@ -91,7 +91,7 @@ compute_resource:
                   - default: ssd
   google_compute_instance_from_template:
     paths:
-      - .planned_values.root_module.resources[] | select(.type == "google_compute_instance_from_template")
+      - select(.type == "google_compute_instance_from_template")
     type: resource
     variables:
       properties:
@@ -100,7 +100,7 @@ compute_resource:
             - '.configuration.root_module.resources[] | select(.address == "${this.address}") | .expressions.source_instance_template.references[] | select(endswith("id")) | gsub("\\.id$"; "")'
             reference:
               paths:
-                - .planned_values.root_module.resources[] | select(.address == "${key}")
+                - select(.address == "${key}")
                 - .planned_values.root_module.child_modules[] | select(.address == ("${key}" | split(".")[0:2] | join("."))) | .resources[] | select(.name == ("${key}" | split(".")[2]))
                 - .prior_state.values.root_module.resources[] | select(.address == "${key}")
               return_path: true

internal/plan/mappings/gcp/compute_group.yaml

@@ -1,8 +1,8 @@
 compute_resource:
   google_compute_instance_group_manager:
     paths:
-      - .planned_values.root_module.resources[] | select(.type == "google_compute_instance_group_manager")
-      - .planned_values.root_module.resources[] | select(.type == "google_compute_region_instance_group_manager")
+      - select(.type == "google_compute_instance_group_manager")
+      - select(.type == "google_compute_region_instance_group_manager")
     type: resource
     variables:
       properties:
@@ -11,17 +11,16 @@ compute_resource:
               - '.configuration.root_module.resources[] | select(.address == "${this.address}") | .expressions.version[0].instance_template.references[] | select(endswith("id")) | gsub("\\.id$"; "")'
             reference:
               paths:
-                - .planned_values.root_module.resources[] | select(.address == "${key}")
-                - .planned_values.root_module.child_modules[].resources | map(select(.address == "${key}"))
+                - select(.address == "${key}")
                 - .prior_state.values.root_module.resources[] | select(.address == "${key}")
               return_path: true
         autoscaler:
           - paths:
             - '(.configuration.root_module.resources[] | select(.expressions.target?.references[]? == "${this.address}") | .address)'
             reference:
               paths:
-                - .planned_values.root_module.resources[] | select(.address == "${key}")
-                - .planned_values.root_module.child_modules[].resources | map(select(.address == "${key}"))
+                - select(.address == "${key}")
+                -  map(select(.address == "${key}"))
                 - .prior_state.values.root_module.resources[] | select(.address == "${key}")
               return_path: true
     properties:

internal/plan/mappings/gcp/disk.yaml

@@ -1,8 +1,8 @@
 compute_resource:
   google_compute_disk:
     paths:
-      - .planned_values.root_module.resources[] | select(.type == "google_compute_disk")
-      - .planned_values.root_module.resources[] | select(.type == "google_compute_region_disk")
+      -  select(.type == "google_compute_disk")
+      -  select(.type == "google_compute_region_disk")
     type: resource
     properties:
       name:

internal/plan/mappings/gcp/sql_database.yaml

@@ -1,6 +1,6 @@
 compute_resource:
   google_sql_database_instance:
-    paths: .planned_values.root_module.resources[] | select(.type == "google_sql_database_instance")
+    paths: select(.type == "google_sql_database_instance")
     type: resource
     properties:
       name:

internal/plan/resolver.go

@@ -6,7 +6,6 @@ import (
 	"regexp"
 
 	"github.com/carboniferio/carbonifer/internal/data"
-	"github.com/carboniferio/carbonifer/internal/utils"
 	"github.com/pkg/errors"
 	"github.com/shopspring/decimal"
 	log "github.com/sirupsen/logrus"
@@ -76,7 +75,7 @@ func resolveReference(key string, reference *Reference, context *tfContext) (int
 			return nil, err
 		}
 		for _, path := range paths {
-			referencedItems, err := utils.GetJSON(path, *TfPlan)
+			referencedItems, err := getJSON(path, *TfPlan)
 			if err != nil {
 				errW := errors.Wrapf(err, "Cannot find referenced path in terraform plan: '%v'", path)
 				return nil, errW

internal/plan/resources.go

@@ -21,19 +21,27 @@ var TfPlan *map[string]interface{}
 func GetResources(tfplan *map[string]interface{}) (map[string]resources.Resource, error) {
 	TfPlan = tfplan
 
+	plannedResources := []interface{}{}
+
 	// Get resources from Terraform plan
-	plannedResourcesResult, err := utils.GetJSON(".planned_values.root_module.resources", *TfPlan)
+	jqPath := ".planned_values | .. | objects | select(has(\"resources\")) | .resources[]"
+	plannedResourcesResult, err := utils.GetJSON(jqPath, *TfPlan)
+
 	if err != nil {
 		return nil, err
 	}
-	if len(plannedResourcesResult) == 0 {
-		return nil, errors.New("No resources found in Terraform plan")
+	if len(plannedResourcesResult) > 0 {
+		for _, plannedResourceI := range plannedResourcesResult {
+			plannedResource, ok := plannedResourceI.(map[string]interface{})
+			if !ok {
+				return nil, errors.Errorf("Cannot parse planned resource %v", plannedResourceI)
+			}
+			plannedResources = append(plannedResources, plannedResource)
+		}
 	}
-	plannedResources := plannedResourcesResult[0].([]interface{})
-	log.Debugf("Reading resources from Terraform plan: %d resources", len(plannedResources))
-	resourcesMap := map[string]resources.Resource{}
 
 	// Get compute resources
+	resourcesMap := map[string]resources.Resource{}
 	mapping, err := GetMapping()
 	if err != nil {
 		errW := errors.Wrap(err, "Cannot get mapping")
@@ -69,6 +77,7 @@ func GetResources(tfplan *map[string]interface{}) (map[string]resources.Resource
 			}
 			unsupportedResource := resources.UnsupportedResource{
 				Identification: &resources.ResourceIdentification{
+					Address:      resourceAddress,
 					Name:         resource["name"].(string),
 					ResourceType: resourceType,
 					Provider:     provider,
@@ -111,7 +120,7 @@ func getResourcesOfType(resourceType string, mapping *ResourceMapping) ([]resour
 	resourcesResult := []resources.Resource{}
 	for _, path := range paths {
 		log.Debugf("  Reading resources of type '%s' from path '%s'", resourceType, path)
-		resourcesFound, err := utils.GetJSON(path, *TfPlan)
+		resourcesFound, err := getJSON(path, *TfPlan)
 		if err != nil {
 			errW := errors.Wrapf(err, "Cannot find resource for path %v", path)
 			return nil, errW
@@ -182,6 +191,7 @@ func GetComputeResource(resourceI interface{}, resourceMapping *ResourceMapping,
 			ResourceType: *resourceType,
 			Provider:     provider,
 			Region:       *region,
+			Address:      resourceAddress,
 		},
 		Specs: &resources.ComputeResourceSpecs{
 			HddStorage:        decimal.Zero,

internal/plan/test/resources_aws_rds_test.go

@@ -27,6 +27,7 @@ func TestGetResource_RDS(t *testing.T) {
 	wantResources := map[string]resources.Resource{
 		"aws_db_instance.first": resources.ComputeResource{
 			Identification: &resources.ResourceIdentification{
+				Address:      "aws_db_instance.first",
 				Name:         "first",
 				ResourceType: "aws_db_instance",
 				Provider:     providers.AWS,
@@ -43,6 +44,7 @@ func TestGetResource_RDS(t *testing.T) {
 		},
 		"aws_db_instance.second": resources.ComputeResource{
 			Identification: &resources.ResourceIdentification{
+				Address:      "aws_db_instance.second",
 				Name:         "second",
 				ResourceType: "aws_db_instance",
 				Provider:     providers.AWS,
@@ -59,6 +61,7 @@ func TestGetResource_RDS(t *testing.T) {
 		},
 		"aws_db_instance.third": resources.ComputeResource{
 			Identification: &resources.ResourceIdentification{
+				Address:      "aws_db_instance.third",
 				Name:         "third",
 				ResourceType: "aws_db_instance",
 				Provider:     providers.AWS,
@@ -78,7 +81,7 @@ func TestGetResource_RDS(t *testing.T) {
 	assert.NoError(t, err)
 	gotResources, err := plan.GetResources(tfPlan)
 	assert.NoError(t, err)
-	for _, res := range gotResources {
-		assert.Equal(t, wantResources[res.GetAddress()], res)
+	for _, got := range gotResources {
+		assert.Equal(t, wantResources[got.GetAddress()], got)
 	}
 }

internal/plan/test/resources_aws_test.go

@@ -28,6 +28,7 @@ func TestGetResource_DiskFromAMI(t *testing.T) {
 		"aws_instance.foo": resources.ComputeResource{
 			Identification: &resources.ResourceIdentification{
 				Name:         "foo",
+				Address:      "aws_instance.foo",
 				ResourceType: "aws_instance",
 				Provider:     providers.AWS,
 				Region:       "eu-west-3",
@@ -43,6 +44,7 @@ func TestGetResource_DiskFromAMI(t *testing.T) {
 		},
 		"aws_ebs_volume.ebs_volume": resources.ComputeResource{
 			Identification: &resources.ResourceIdentification{
+				Address:      "aws_ebs_volume.ebs_volume",
 				Name:         "ebs_volume",
 				ResourceType: "aws_ebs_volume",
 				Provider:     providers.AWS,
@@ -57,6 +59,7 @@ func TestGetResource_DiskFromAMI(t *testing.T) {
 		},
 		"aws_network_interface.foo": resources.UnsupportedResource{
 			Identification: &resources.ResourceIdentification{
+				Address:      "aws_network_interface.foo",
 				Name:         "foo",
 				ResourceType: "aws_network_interface",
 				Provider:     providers.AWS,
@@ -65,6 +68,7 @@ func TestGetResource_DiskFromAMI(t *testing.T) {
 		},
 		"aws_subnet.my_subnet": resources.UnsupportedResource{
 			Identification: &resources.ResourceIdentification{
+				Address:      "aws_subnet.my_subnet",
 				Name:         "my_subnet",
 				ResourceType: "aws_subnet",
 				Provider:     providers.AWS,