Add support for the custom date range metric data in the table aws_cost_usage_* table

aws/cost_explorer.go

@@ -2,12 +2,12 @@ package aws
 
 import (
 	"context"
+	"strings"
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/service/costexplorer"
 	"github.com/aws/aws-sdk-go-v2/service/costexplorer/types"
-	"github.com/golang/protobuf/ptypes/timestamp"
 
 	"github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto"
 	"github.com/turbot/steampipe-plugin-sdk/v5/plugin"
@@ -26,6 +26,32 @@ func AllCostMetrics() []string {
 	}
 }
 
+// getCostMetricByMetricName returns the select metrics
+func getCostMetricByMetricName(metricName string) []string {
+	metrics := strings.Split(metricName, ",")
+	var selectedMetric []string
+	for _, m := range metrics {
+		switch strings.ToLower(m) {
+		case "blendedcost":
+			selectedMetric = append(selectedMetric, "BlendedCost")
+		case "unblendedcost":
+			selectedMetric = append(selectedMetric, "UnblendedCost")
+		case "netunblendedcost":
+			selectedMetric = append(selectedMetric, "NetUnblendedCost")
+		case "amortizedcost":
+			selectedMetric = append(selectedMetric, "AmortizedCost")
+		case "netamortizedcost":
+			selectedMetric = append(selectedMetric, "NetAmortizedCost")
+		case "usagequantity":
+			selectedMetric = append(selectedMetric, "UsageQuantity")
+		case "normalizedusageamount":
+			selectedMetric = append(selectedMetric, "NormalizedUsageAmount")
+		}
+	}
+
+	return selectedMetric
+}
+
 var costExplorerColumnDefs = []*plugin.Column{
 
 	{
@@ -127,6 +153,30 @@ func costExplorerColumns(columns []*plugin.Column) []*plugin.Column {
 	return append(columns, costExplorerColumnDefs...)
 }
 
+// append search timestamp columns
+func searchByTimeAndMetricColumns(otherColumns []*plugin.Column) []*plugin.Column {
+	return append([]*plugin.Column{
+		{
+			Name:        "search_start_time",
+			Description: "Search start timestamp for this cost metric.",
+			Type:        proto.ColumnType_TIMESTAMP,
+			Hydrate:     hydrateCostAndUsageQuals,
+		},
+		{
+			Name:        "search_end_time",
+			Description: "Search end timestamp for this cost metric.",
+			Type:        proto.ColumnType_TIMESTAMP,
+			Hydrate:     hydrateCostAndUsageQuals,
+		},
+		{
+			Name:        "metrics",
+			Description: "This cost metrics.",
+			Type:        proto.ColumnType_STRING,
+			Hydrate:     hydrateCostAndUsageQuals,
+		},
+	}, otherColumns...)
+}
+
 //// LIST FUNCTION
 
 func streamCostAndUsage(ctx context.Context, d *plugin.QueryData, params *costexplorer.GetCostAndUsageInput) (interface{}, error) {
@@ -268,25 +318,71 @@ func getCEStartDateForGranularity(granularity string) time.Time {
 
 type CEQuals struct {
 	// Quals stuff
-	SearchStartTime *timestamp.Timestamp
-	SearchEndTime   *timestamp.Timestamp
+	SearchStartTime *time.Time
+	SearchEndTime   *time.Time
+	Metrics         string
 	Granularity     string
 	DimensionType1  string
 	DimensionType2  string
 	TagKey1         string
 	TagKey2         string
 }
 
-func hydrateCostAndUsageQuals(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateData) (interface{}, error) {
+func hydrateCostAndUsageQuals(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) {
 	plugin.Logger(ctx).Debug("hydrateKeyQuals", "d.EqualsQuals", d.EqualsQuals)
 
 	return &CEQuals{
-		SearchStartTime: d.EqualsQuals["search_start_time"].GetTimestampValue(),
-		SearchEndTime:   d.EqualsQuals["search_end_time"].GetTimestampValue(),
+		SearchStartTime: getSearchTimestampValueFromQuals(ctx, d, "search_start_time", h),
+		SearchEndTime:   getSearchTimestampValueFromQuals(ctx, d, "search_end_time", h),
+		Metrics:         d.EqualsQuals["metrics"].GetStringValue(),
 		Granularity:     d.EqualsQuals["granularity"].GetStringValue(),
 		DimensionType1:  d.EqualsQuals["dimension_type_1"].GetStringValue(),
 		DimensionType2:  d.EqualsQuals["dimension_type_2"].GetStringValue(),
 		TagKey1:         d.EqualsQuals["tag_key_1"].GetStringValue(),
 		TagKey2:         d.EqualsQuals["tag_key_2"].GetStringValue(),
 	}, nil
 }
+
+// In the case of >, <=, <, or <= operator uses in the where clause.
+// The value for the column search_start_time and search_end_time are not being populated correctly
+// by 'd.EqualsQuals["search_end_time"].GetTimestampValue().AsTime()'.
+func getSearchTimestampValueFromQuals(ctx context.Context, quals *plugin.QueryData, columnName string, h *plugin.HydrateData) *time.Time {
+	var st, et *string
+	if h.Item != nil {
+		switch item := h.Item.(type) {
+		case CEMetricRow:
+			st = item.PeriodStart
+			et = item.PeriodEnd
+		case types.ForecastResult:
+			st = item.TimePeriod.Start
+			et = item.TimePeriod.End
+		}
+	}
+
+	if quals.Quals[columnName] != nil {
+		var t time.Time
+		for _, q := range quals.Quals[columnName].Quals {
+			if q.Operator == ">" || q.Operator == "<" {
+				if columnName == "search_start_time" {
+					t = convertStringTypeToTimestamp(*st)
+				}
+				if columnName == "search_end_time" {
+					t = convertStringTypeToTimestamp(*et)
+				}
+			} else {
+				t = q.Value.GetTimestampValue().AsTime()
+			}
+		}
+		return &t
+	}
+	return nil
+}
+
+func convertStringTypeToTimestamp(t string) time.Time {
+	timeFormat := "2006-01-02"
+	ti, err := time.Parse(timeFormat, t)
+	if err != nil {
+		panic("Error parsing search time: " + err.Error())
+	}
+	return ti
+}

aws/table_aws_cost_by_account_daily.go

@@ -14,25 +14,46 @@ func tableAwsCostByLinkedAccountDaily(_ context.Context) *plugin.Table {
 		Description: "AWS Cost Explorer - Cost by Linked Account (Daily)",
 		List: &plugin.ListConfig{
 			Hydrate: listCostByLinkedAccountDaily,
-			Tags:    map[string]string{"service": "ce", "action": "GetCostAndUsage"},
-		},
-		Columns: awsGlobalRegionColumns(
-			costExplorerColumns([]*plugin.Column{
-
+			KeyColumns: plugin.KeyColumnSlice{
+				{
+					Name:      "metrics",
+					Require:   plugin.Optional,
+					Operators: []string{"="},
+					CacheMatch: "exact",
+				},
 				{
-					Name:        "linked_account_id",
-					Description: "The AWS Account ID.",
-					Type:        proto.ColumnType_STRING,
-					Transform:   transform.FromField("Dimension1"),
+					Name:       "search_start_time",
+					Require:    plugin.Optional,
+					Operators:  []string{">", ">=", "=", "<", "<="},
+					CacheMatch: "exact",
 				},
-			}),
+				{
+					Name:       "search_end_time",
+					Require:    plugin.Optional,
+					Operators:  []string{">", ">=", "=", "<", "<="},
+					CacheMatch: "exact",
+				},
+			},
+			Tags: map[string]string{"service": "ce", "action": "GetCostAndUsage"},
+		},
+		Columns: awsGlobalRegionColumns(
+			costExplorerColumns(
+				searchByTimeAndMetricColumns([]*plugin.Column{
+					{
+						Name:        "linked_account_id",
+						Description: "The AWS Account ID.",
+						Type:        proto.ColumnType_STRING,
+						Transform:   transform.FromField("Dimension1"),
+					},
+				}),
+			),
 		),
 	}
 }
 
 //// LIST FUNCTION
 
 func listCostByLinkedAccountDaily(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateData) (interface{}, error) {
-	params := buildCostByLinkedAccountInput("DAILY")
+	params := buildCostByLinkedAccountInput(d, "DAILY")
 	return streamCostAndUsage(ctx, d, params)
 }

aws/table_aws_cost_by_account_monthly.go

@@ -2,6 +2,8 @@ package aws
 
 import (
 	"context"
+	"fmt"
+	"strings"
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
@@ -19,45 +21,83 @@ func tableAwsCostByLinkedAccountMonthly(_ context.Context) *plugin.Table {
 		Description: "AWS Cost Explorer - Cost by Linked Account (Monthly)",
 		List: &plugin.ListConfig{
 			Hydrate: listCostByLinkedAccountMonthly,
-			Tags:    map[string]string{"service": "ce", "action": "GetCostAndUsage"},
-		},
-		Columns: awsGlobalRegionColumns(
-			costExplorerColumns([]*plugin.Column{
-
+			KeyColumns: plugin.KeyColumnSlice{
+				{
+					Name:       "metrics",
+					Require:    plugin.Optional,
+					Operators:  []string{"="},
+					CacheMatch: "exact",
+				},
 				{
-					Name:        "linked_account_id",
-					Description: "The AWS Account ID.",
-					Type:        proto.ColumnType_STRING,
-					Transform:   transform.FromField("Dimension1"),
+					Name:       "search_start_time",
+					Require:    plugin.Optional,
+					Operators:  []string{">", ">=", "=", "<", "<="},
+					CacheMatch: "exact",
 				},
-			}),
+				{
+					Name:       "search_end_time",
+					Require:    plugin.Optional,
+					Operators:  []string{">", ">=", "=", "<", "<="},
+					CacheMatch: "exact",
+				},
+			},
+			Tags: map[string]string{"service": "ce", "action": "GetCostAndUsage"},
+		},
+		Columns: awsGlobalRegionColumns(
+			costExplorerColumns(
+				searchByTimeAndMetricColumns([]*plugin.Column{
+					{
+						Name:        "linked_account_id",
+						Description: "The AWS Account ID.",
+						Type:        proto.ColumnType_STRING,
+						Transform:   transform.FromField("Dimension1"),
+					},
+				}),
+			),
 		),
 	}
 }
 
 //// LIST FUNCTION
 
 func listCostByLinkedAccountMonthly(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateData) (interface{}, error) {
-	params := buildCostByLinkedAccountInput("MONTHLY")
-
+	params := buildCostByLinkedAccountInput(d, "MONTHLY")
 	return streamCostAndUsage(ctx, d, params)
 }
 
-func buildCostByLinkedAccountInput(granularity string) *costexplorer.GetCostAndUsageInput {
+func buildCostByLinkedAccountInput(d *plugin.QueryData, granularity string) *costexplorer.GetCostAndUsageInput {
 	timeFormat := "2006-01-02"
 	if granularity == "HOURLY" {
 		timeFormat = "2006-01-02T15:04:05Z"
 	}
 	endTime := time.Now().Format(timeFormat)
 	startTime := getCEStartDateForGranularity(granularity).Format(timeFormat)
 
+	st, et := getSearchStartTImeAndSearchEndTime(d, granularity)
+	if st != "" {
+		startTime = st
+	}
+	if et != "" {
+		endTime = et
+	}
+
+	selectedMetrics := AllCostMetrics()
+	if d.EqualsQualString("metrics") != "" {
+		m := getCostMetricByMetricName(d.EqualsQualString("metrics"))
+		if !(len(m) > 0) {
+			panic(fmt.Sprintf("unsupported metric '%s', supported metrics are %s", d.EqualsQualString("metrics"), strings.Join(selectedMetrics, ",")))
+		}
+
+		selectedMetrics = m
+	}
+
 	params := &costexplorer.GetCostAndUsageInput{
 		TimePeriod: &types.DateInterval{
 			Start: aws.String(startTime),
 			End:   aws.String(endTime),
 		},
 		Granularity: types.Granularity(granularity),
-		Metrics:     AllCostMetrics(),
+		Metrics:     selectedMetrics,
 		GroupBy: []types.GroupDefinition{
 			{
 				Type: types.GroupDefinitionType("DIMENSION"),

aws/table_aws_cost_by_record_type_daily.go

@@ -14,31 +14,52 @@ func tableAwsCostByRecordTypeDaily(_ context.Context) *plugin.Table {
 		Description: "AWS Cost Explorer - Cost by Record Type (Daily)",
 		List: &plugin.ListConfig{
 			Hydrate: listCostByRecordTypeDaily,
-			Tags:    map[string]string{"service": "ce", "action": "GetCostAndUsage"},
-		},
-		Columns: awsGlobalRegionColumns(
-			costExplorerColumns([]*plugin.Column{
-
+			KeyColumns: plugin.KeyColumnSlice{
 				{
-					Name:        "linked_account_id",
-					Description: "The linked AWS Account ID.",
-					Type:        proto.ColumnType_STRING,
-					Transform:   transform.FromField("Dimension1"),
+					Name:      "metrics",
+					Require:   plugin.Optional,
+					Operators: []string{"="},
+					CacheMatch: "exact",
 				},
 				{
-					Name:        "record_type",
-					Description: "The different types of charges such as RI fees, usage, costs, tax refunds, and credits.",
-					Type:        proto.ColumnType_STRING,
-					Transform:   transform.FromField("Dimension2"),
+					Name:       "search_start_time",
+					Require:    plugin.Optional,
+					Operators:  []string{">", ">=", "=", "<", "<="},
+					CacheMatch: "exact",
 				},
-			}),
+				{
+					Name:       "search_end_time",
+					Require:    plugin.Optional,
+					Operators:  []string{">", ">=", "=", "<", "<="},
+					CacheMatch: "exact",
+				},
+			},
+			Tags: map[string]string{"service": "ce", "action": "GetCostAndUsage"},
+		},
+		Columns: awsGlobalRegionColumns(
+			costExplorerColumns(
+				searchByTimeAndMetricColumns([]*plugin.Column{
+					{
+						Name:        "linked_account_id",
+						Description: "The linked AWS Account ID.",
+						Type:        proto.ColumnType_STRING,
+						Transform:   transform.FromField("Dimension1"),
+					},
+					{
+						Name:        "record_type",
+						Description: "The different types of charges such as RI fees, usage, costs, tax refunds, and credits.",
+						Type:        proto.ColumnType_STRING,
+						Transform:   transform.FromField("Dimension2"),
+					},
+				}),
+			),
 		),
 	}
 }
 
 //// LIST FUNCTION
 
 func listCostByRecordTypeDaily(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateData) (interface{}, error) {
-	params := buildCostByRecordTypeInput("DAILY")
+	params := buildCostByRecordTypeInput(d, "DAILY")
 	return streamCostAndUsage(ctx, d, params)
 }