From e48be483226b5ef452eb4d47a38525d32c0407a5 Mon Sep 17 00:00:00 2001 From: ParthaI Date: Fri, 6 Sep 2024 17:30:56 +0530 Subject: [PATCH 1/3] Optimize API call, query timing for the table aws_ec2_instance_type and added support to accept the wildcard pattern for instance type. Closes #2292 --- aws/table_aws_ec2_instance_type.go | 209 +++++++++++++++------------ docs/tables/aws_ec2_instance_type.md | 6 + 2 files changed, 122 insertions(+), 93 deletions(-) diff --git a/aws/table_aws_ec2_instance_type.go b/aws/table_aws_ec2_instance_type.go index 8d0665ddd..929d50300 100644 --- a/aws/table_aws_ec2_instance_type.go +++ b/aws/table_aws_ec2_instance_type.go @@ -3,6 +3,8 @@ package aws import ( "context" "fmt" + "regexp" + "strings" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ec2" @@ -13,6 +15,7 @@ import ( "github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto" "github.com/turbot/steampipe-plugin-sdk/v5/plugin" "github.com/turbot/steampipe-plugin-sdk/v5/plugin/transform" + "github.com/turbot/steampipe-plugin-sdk/v5/query_cache" ) //// TABLE DEFINITION @@ -21,29 +24,14 @@ func tableAwsInstanceType(_ context.Context) *plugin.Table { return &plugin.Table{ Name: "aws_ec2_instance_type", Description: "AWS EC2 Instance Type", - Get: &plugin.GetConfig{ - // We must have to include the region in the query parameter to make the gate API call. - // Otherwise we will get an Error: get call returned 9 results - the key column is not globally unique (SQLSTATE HV000) - KeyColumns: plugin.AllColumns([]string{"instance_type", "region"}), - IgnoreConfig: &plugin.IgnoreConfig{ - ShouldIgnoreErrorFunc: shouldIgnoreErrors([]string{"InvalidInstanceType"}), - }, - Hydrate: describeInstanceType, - Tags: map[string]string{"service": "ec2", "action": "DescribeInstanceTypes"}, - }, List: &plugin.ListConfig{ Hydrate: listAwsInstanceTypesOfferings, KeyColumns: plugin.KeyColumnSlice{ {Name: "instance_type", Require: plugin.Optional, Operators: []string{"="}}, + {Name: "instance_type_pattern", Require: plugin.Optional, Operators: []string{"="}, CacheMatch: query_cache.CacheMatchExact}, }, Tags: map[string]string{"service": "ec2", "action": "DescribeInstanceTypeOfferings"}, }, - HydrateConfig: []plugin.HydrateConfig{ - { - Func: describeInstanceType, - Tags: map[string]string{"service": "ec2", "action": "DescribeInstanceTypes"}, - }, - }, GetMatrixItemFunc: SupportedRegionMatrix(ec2v1.EndpointsID), Columns: awsRegionalColumns([]*plugin.Column{ { @@ -51,77 +39,74 @@ func tableAwsInstanceType(_ context.Context) *plugin.Table { Description: "The instance type. For more information, see [ Instance Types ](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html) in the Amazon Elastic Compute Cloud User Guide.", Type: proto.ColumnType_STRING, }, + // In the query "select * from aws_ec2_instance_type where instance_type = 't2*'", the API fetches the result but returns empty rows due to PostgreSQL-level filtering. + // The 'instance_type' column contains values like 't2.small', not 't2*', leading to a mismatch between the column value ('t2.small') and the wildcard pattern used in the query ('t2*'). + // To resolve this issue, the 'instance_type_pattern' column has been added, allowing for proper filtering using wildcard patterns. + { + Name: "instance_type_pattern", + Description: "The instance type pattern includes wildcards, such as 'c5-*', 't2*', and 'm5*'.", + Type: proto.ColumnType_STRING, + Transform: transform.FromQual("instance_type_pattern"), + }, { Name: "auto_recovery_supported", Description: "Indicates whether auto recovery is supported.", Type: proto.ColumnType_BOOL, - Hydrate: describeInstanceType, }, { Name: "bare_metal", Description: "Indicates whether the instance is a bare metal instance type.", Type: proto.ColumnType_BOOL, - Hydrate: describeInstanceType, }, { Name: "burstable_performance_supported", Description: "Indicates whether the instance type is a burstable performance instance type.", Type: proto.ColumnType_BOOL, - Hydrate: describeInstanceType, }, { Name: "current_generation", Description: "Indicates whether the instance type is current generation.", Type: proto.ColumnType_BOOL, - Hydrate: describeInstanceType, }, { Name: "dedicated_hosts_supported", Description: "Indicates whether Dedicated Hosts are supported on the instance type.", Type: proto.ColumnType_BOOL, - Hydrate: describeInstanceType, }, { Name: "free_tier_eligible", Description: "Indicates whether the instance type is eligible for the free tier.", Type: proto.ColumnType_BOOL, - Hydrate: describeInstanceType, }, { Name: "nitro_enclaves_support", Description: "Indicates whether Nitro Enclaves is supported.", Type: proto.ColumnType_STRING, - Hydrate: describeInstanceType, }, { Name: "nitro_tpm_support", Description: "Indicates whether NitroTPM is supported.", Type: proto.ColumnType_STRING, - Hydrate: describeInstanceType, }, { Name: "hibernation_supported", Description: "Indicates whether On-Demand hibernation is supported.", Type: proto.ColumnType_BOOL, - Hydrate: describeInstanceType, }, { Name: "hypervisor", Description: "The hypervisor for the instance type.", Type: proto.ColumnType_STRING, - Hydrate: describeInstanceType, }, { Name: "instance_storage_supported", Description: "Describes the instance storage for the instance type.", Type: proto.ColumnType_STRING, - Hydrate: describeInstanceType, }, { Name: "ebs_info", Description: "Describes the Amazon EBS settings for the instance type.", Type: proto.ColumnType_JSON, - Hydrate: describeInstanceType, }, { Name: "location_type", @@ -132,104 +117,87 @@ func tableAwsInstanceType(_ context.Context) *plugin.Table { Name: "memory_info", Description: "Describes the memory for the instance type.", Type: proto.ColumnType_JSON, - Hydrate: describeInstanceType, }, { Name: "network_info", Description: "Describes the network settings for the instance type.", Type: proto.ColumnType_JSON, - Hydrate: describeInstanceType, }, { Name: "placement_group_info", Description: "Describes the placement group settings for the instance type.", Type: proto.ColumnType_JSON, - Hydrate: describeInstanceType, }, { Name: "processor_info", Description: "Describes the processor.", Type: proto.ColumnType_JSON, - Hydrate: describeInstanceType, }, { Name: "supported_root_device_types", Description: "The supported root device types.", Type: proto.ColumnType_JSON, - Hydrate: describeInstanceType, }, { Name: "supported_usage_classes", Description: "Indicates whether the instance type is offered for spot or On-Demand.", Type: proto.ColumnType_JSON, - Hydrate: describeInstanceType, }, { Name: "supported_virtualization_types", Description: "The supported virtualization types.", Type: proto.ColumnType_JSON, - Hydrate: describeInstanceType, }, { Name: "v_cpu_info", Description: "Describes the vCPU configurations for the instance type.", Type: proto.ColumnType_JSON, - Hydrate: describeInstanceType, Transform: transform.FromField("VCpuInfo"), }, { Name: "gpu_info", Description: "Describes the GPU accelerator settings for the instance type.", Type: proto.ColumnType_JSON, - Hydrate: describeInstanceType, }, { Name: "fpga_info", Description: "Describes the FPGA accelerator settings for the instance type.", Type: proto.ColumnType_JSON, - Hydrate: describeInstanceType, }, { Name: "inference_accelerator_info", Description: "Describes the Inference accelerator settings for the instance type.", Type: proto.ColumnType_JSON, - Hydrate: describeInstanceType, }, { Name: "instance_storage_info", Description: "Describes the instance storage for the instance type.", Type: proto.ColumnType_JSON, - Hydrate: describeInstanceType, }, { Name: "media_accelerator_info", Description: "Describes the media accelerator settings for the instance type.", Type: proto.ColumnType_JSON, - Hydrate: describeInstanceType, }, { Name: "neuron_info", Description: "Describes the Neuron accelerator settings for the instance type.", Type: proto.ColumnType_JSON, - Hydrate: describeInstanceType, }, { Name: "nitro_tpm_info", Description: "Describes the supported NitroTPM versions for the instance type.", Type: proto.ColumnType_JSON, - Hydrate: describeInstanceType, }, { Name: "supported_boot_modes", Description: "The supported boot modes.", Type: proto.ColumnType_JSON, - Hydrate: describeInstanceType, }, { Name: "title", Description: resourceInterfaceDescription("title"), Type: proto.ColumnType_STRING, - Hydrate: describeInstanceType, Transform: transform.FromField("InstanceType"), }, { @@ -248,18 +216,19 @@ func tableAwsInstanceType(_ context.Context) *plugin.Table { func listAwsInstanceTypesOfferings(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { region := d.EqualsQualString(matrixKeyRegion) - // Create Session + // Create EC2 client svc, err := EC2Client(ctx, d) if err != nil { plugin.Logger(ctx).Error("aws_ec2_instance_type.listAwsInstanceTypesOfferings", "connection_error", err) return nil, err } + + // If the service is nil, the region is unsupported if svc == nil { - // Unsupported region, return no data return nil, nil } - // Limiting the results + // Set the maximum limit for results maxLimit := int32(1000) if d.QueryContext.Limit != nil { limit := int32(*d.QueryContext.Limit) @@ -272,99 +241,153 @@ func listAwsInstanceTypesOfferings(ctx context.Context, d *plugin.QueryData, h * } } - // First get all the types of instance + // Prepare the input for EC2 DescribeInstanceTypeOfferings API input := &ec2.DescribeInstanceTypeOfferingsInput{ LocationType: types.LocationTypeRegion, MaxResults: aws.Int32(maxLimit), + Filters: []types.Filter{{Name: aws.String("location"), Values: []string{region}}}, } - var filters []types.Filter - filters = append(filters, types.Filter{Name: aws.String("location"), Values: []string{region}}) + // Fetch instance type offering for a particular instance type if d.EqualsQualString("instance_type") != "" { - filters = append(filters, types.Filter{Name: aws.String("instance-type"), Values: []string{d.EqualsQualString("instance_type")}}) + input.Filters = append(input.Filters, types.Filter{Name: aws.String("instance-type"), Values: []string{d.EqualsQualString("instance_type")}}) } - input.Filters = filters + // Fetch instance types offerings + instanceTypes, err := fetchInstanceTypeOfferings(ctx, d, svc, input, maxLimit) + if err != nil { + return nil, err + } + + // Apply pattern matching on instance types if provided in query + filteredInstanceTypes := filterInstanceTypesByPattern(ctx, instanceTypes, d.EqualsQualString("instance_type_pattern")) + + // Batch process the instance types in groups of 100 + err = batchDescribeInstanceTypes(ctx, d, filteredInstanceTypes, region) + if err != nil { + plugin.Logger(ctx).Error("aws_ec2_instance_type.listAwsInstanceTypesOfferings", "batch_process_error", err) + return nil, err + } + + return nil, nil +} + +// Helper function to fetch instance type offerings using pagination +func fetchInstanceTypeOfferings(ctx context.Context, d *plugin.QueryData, svc *ec2.Client, input *ec2.DescribeInstanceTypeOfferingsInput, maxLimit int32) ([]types.InstanceType, error) { + var instanceTypes []types.InstanceType paginator := ec2.NewDescribeInstanceTypeOfferingsPaginator(svc, input, func(o *ec2.DescribeInstanceTypeOfferingsPaginatorOptions) { o.Limit = maxLimit o.StopOnDuplicateToken = true }) - // List call for paginator.HasMorePages() { - // apply rate limiting d.WaitForListRateLimit(ctx) output, err := paginator.NextPage(ctx) if err != nil { - plugin.Logger(ctx).Error("aws_ec2_instance_type.listAwsInstanceTypesOfferings", "api_error", err) + plugin.Logger(ctx).Error("aws_ec2_instance_type.fetchInstanceTypeOfferings", "api_error", err) return nil, err } - for _, items := range output.InstanceTypeOfferings { - d.StreamListItem(ctx, items) + for _, item := range output.InstanceTypeOfferings { + instanceTypes = append(instanceTypes, item.InstanceType) + } + } + + return instanceTypes, nil +} - // Context can be cancelled due to manual cancellation or the limit has been hit - if d.RowsRemaining(ctx) == 0 { - return nil, nil - } +// Helper function to filter instance types by a pattern like t2-*, m5-*, etc. +func filterInstanceTypesByPattern(ctx context.Context, instanceTypes []types.InstanceType, pattern string) []types.InstanceType { + plugin.Logger(ctx).Error("Pattern =>>>", pattern) + if pattern == "" { + return instanceTypes + } + + // The regex pattern "t3*" does not work as expected when matching the string "t3.small". This is because '*' in regex matches zero or more occurrences of the preceding character, + // allowing it to match "t3.small" due to the presence of the '.' character. To correct this, we replace '*' with '.+' to match any characters after "t3" appropriately. + // The following patterns were validated: + // - "c7*" matches (e.g., c7i.2xlarge, c7gn.xlarge, c7i-flex.2xlarge), + // - "c7i*" matches (e.g., c7i.2xlarge, c7i-flex.2xlarge), + // - "c7i.*" matches (e.g., c7i.8xlarge, c7i.2xlarge), + // - "c7i-*" matches (e.g., c7i-flex.2xlarge). + pattern = strings.ReplaceAll(pattern, ".", "\\.") + pattern = strings.ReplaceAll(pattern, "*", ".+") + var matchedInstanceTypes []types.InstanceType + re := regexp.MustCompile(pattern) + + for _, instanceType := range instanceTypes { + + if re.MatchString(string(instanceType)) { + plugin.Logger(ctx).Error("Pattern matched with =>>>", instanceType) + matchedInstanceTypes = append(matchedInstanceTypes, instanceType) } } - return nil, err + return matchedInstanceTypes } -//// HYDRATE FUNCTIONS +// Helper function to batch describe instance types in groups of 100 +func batchDescribeInstanceTypes(ctx context.Context, d *plugin.QueryData, instanceTypes []types.InstanceType, region string) error { + batchSize := 100 + for i := 0; i < len(instanceTypes); i += batchSize { + end := i + batchSize + if end > len(instanceTypes) { + end = len(instanceTypes) + } + batch := instanceTypes[i:end] -func describeInstanceType(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { - var instanceType types.InstanceType - if h.Item != nil { - data := h.Item.(types.InstanceTypeOffering) - instanceType = data.InstanceType - } else { - instanceType = types.InstanceType(d.EqualsQuals["instance_type"].GetStringValue()) + err := describeInstanceTypes(ctx, d, batch, region) + if err != nil { + return err + } } - // Create Session - svc, err := EC2Client(ctx, d) + return nil +} + +// Describe instance types and stream the results +func describeInstanceTypes(ctx context.Context, d *plugin.QueryData, instanceTypes []types.InstanceType, region string) error { + svc, err := EC2ClientForRegion(ctx, d, region) if err != nil { - plugin.Logger(ctx).Error("aws_ec2_instance_type.describeInstanceType", "connection_error", err) - return nil, err + plugin.Logger(ctx).Error("aws_ec2_instance_type.describeInstanceTypes", "connection_error", err) + return err } if svc == nil { // Unsupported region, return no data - return nil, nil + return nil } - // First get all the types of + // Create input for DescribeInstanceTypes API params := &ec2.DescribeInstanceTypesInput{ - InstanceTypes: []types.InstanceType{ - instanceType, - }, + InstanceTypes: instanceTypes, } - // execute get call + // Fetch the instance types op, err := svc.DescribeInstanceTypes(ctx, params) if err != nil { - plugin.Logger(ctx).Error("aws_ec2_instance_type.describeInstanceType", "api_error", err) - return nil, err + plugin.Logger(ctx).Error("aws_ec2_instance_type.describeInstanceTypes", "api_error", err) + return err } - if len(op.InstanceTypes) > 0 { - return op.InstanceTypes[0], nil + + // Stream each item from the response + for _, item := range op.InstanceTypes { + d.StreamListItem(ctx, item) + + // Context may get cancelled due to manual cancellation or if the limit has been reached + if d.RowsRemaining(ctx) == 0 { + return nil + } } - return nil, nil + return nil } +//// HYDRATE FUNCTION + func instanceTypeDataToAkas(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { - var instanceType types.InstanceType - switch h.Item.(type) { - case types.InstanceTypeOffering: - instanceType = h.Item.(types.InstanceTypeOffering).InstanceType - case types.InstanceTypeInfo: - instanceType = h.Item.(types.InstanceTypeInfo).InstanceType - } + instanceType := h.Item.(types.InstanceTypeInfo).InstanceType commonData, err := getCommonColumns(ctx, d, h) if err != nil { diff --git a/docs/tables/aws_ec2_instance_type.md b/docs/tables/aws_ec2_instance_type.md index 33ca38b9c..9e65f6114 100644 --- a/docs/tables/aws_ec2_instance_type.md +++ b/docs/tables/aws_ec2_instance_type.md @@ -11,6 +11,12 @@ The AWS EC2 Instance Type is a component of Amazon's Elastic Compute Cloud (EC2) The `aws_ec2_instance_type` table in Steampipe provides you with information about EC2 instance types within AWS Elastic Compute Cloud (EC2). This table allows you, as a DevOps engineer, to query instance type-specific details, including its name, current generation, vCPU, memory, storage, and network performance. You can utilize this table to gather insights on instance types, such as their capabilities, performance, and associated metadata. The schema outlines the various attributes of the EC2 instance type for you, including the instance type, current generation, vCPU, memory, storage, and network performance. +**Important Notes** +- This table supports the optional quals `instance_type` and `instance_type_pattern`. +- Queries with optional quals are optimised to use additional filtering provided by the AWS API function. +- To filter by a specific `instance_type`, you need to include it in the WHERE clause, such as `where instance_type = 't2.small'`, to retrieve a single instance type. +- If you want to fetch instance types using a wildcard pattern, you can use `instance_type_pattern` in the WHERE clause, like `where instance_type_pattern = 't2*'`. + ## Examples ### List of instance types which supports dedicated host From 87259d22b58a53be132df3bec1fe5369c1061346 Mon Sep 17 00:00:00 2001 From: ParthaI <47887552+ParthaI@users.noreply.github.com> Date: Mon, 18 Nov 2024 15:50:44 +0700 Subject: [PATCH 2/3] Removed unnecessary log statement --- aws/table_aws_ec2_instance_type.go | 1 - 1 file changed, 1 deletion(-) diff --git a/aws/table_aws_ec2_instance_type.go b/aws/table_aws_ec2_instance_type.go index 929d50300..14fdbf672 100644 --- a/aws/table_aws_ec2_instance_type.go +++ b/aws/table_aws_ec2_instance_type.go @@ -300,7 +300,6 @@ func fetchInstanceTypeOfferings(ctx context.Context, d *plugin.QueryData, svc *e // Helper function to filter instance types by a pattern like t2-*, m5-*, etc. func filterInstanceTypesByPattern(ctx context.Context, instanceTypes []types.InstanceType, pattern string) []types.InstanceType { - plugin.Logger(ctx).Error("Pattern =>>>", pattern) if pattern == "" { return instanceTypes } From aa978e0bca242da0c183bee1b9544be6e984491d Mon Sep 17 00:00:00 2001 From: ParthaI Date: Mon, 18 Nov 2024 17:26:33 +0530 Subject: [PATCH 3/3] Optimize the API call based on limit provided in the query parameter --- aws/table_aws_ec2_instance_type.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/aws/table_aws_ec2_instance_type.go b/aws/table_aws_ec2_instance_type.go index 14fdbf672..25d1d78e3 100644 --- a/aws/table_aws_ec2_instance_type.go +++ b/aws/table_aws_ec2_instance_type.go @@ -299,7 +299,7 @@ func fetchInstanceTypeOfferings(ctx context.Context, d *plugin.QueryData, svc *e } // Helper function to filter instance types by a pattern like t2-*, m5-*, etc. -func filterInstanceTypesByPattern(ctx context.Context, instanceTypes []types.InstanceType, pattern string) []types.InstanceType { +func filterInstanceTypesByPattern(_ context.Context, instanceTypes []types.InstanceType, pattern string) []types.InstanceType { if pattern == "" { return instanceTypes } @@ -317,9 +317,7 @@ func filterInstanceTypesByPattern(ctx context.Context, instanceTypes []types.Ins re := regexp.MustCompile(pattern) for _, instanceType := range instanceTypes { - if re.MatchString(string(instanceType)) { - plugin.Logger(ctx).Error("Pattern matched with =>>>", instanceType) matchedInstanceTypes = append(matchedInstanceTypes, instanceType) } } @@ -330,6 +328,9 @@ func filterInstanceTypesByPattern(ctx context.Context, instanceTypes []types.Ins // Helper function to batch describe instance types in groups of 100 func batchDescribeInstanceTypes(ctx context.Context, d *plugin.QueryData, instanceTypes []types.InstanceType, region string) error { batchSize := 100 + if d.QueryContext.Limit != nil && *d.QueryContext.Limit > 0 && *d.QueryContext.Limit < 100 { + batchSize = int(*d.QueryContext.Limit) + } for i := 0; i < len(instanceTypes); i += batchSize { end := i + batchSize if end > len(instanceTypes) {