From 196ade03584cfe1083860a3641b3034c8455fe00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan-Christoph=20K=C3=BCster?= Date: Sat, 19 Dec 2020 21:06:25 +0100 Subject: [PATCH] Refactor generator (#22) * Set default region and profile for generator * Refactor generate list * Discard TRACE logs of Terraform provider * Refactor GetResourceID into method * Refactor a lot * Add some new resources via manual map --- README.md | 10 +- aws/aws_cloudtrail.go | 35 ++ aws/aws_instance.go | 4 +- aws/aws_workspaces_directory.go | 37 ++ ...s_alias.go => aws_workspaces_workspace.go} | 14 +- aws/client.go | 1 + aws/list.go | 6 + gen/aws/README.tpl.md | 112 +++++ gen/aws/client.go | 17 +- gen/aws/list.go | 453 +++++++----------- gen/aws/manual_mapping.go | 31 +- gen/aws/operation.go | 108 ----- gen/aws/readme.go | 54 +-- gen/aws/service.go | 36 +- gen/aws/supported.go | 23 +- gen/aws/util.go | 244 ++++++++++ gen/aws/util_test.go | 29 ++ gen/main.go | 117 +++-- gen/terraform/list_resources.go | 19 +- gen/terraform/resource_id.go | 21 +- gen/terraform/resource_service.go | 24 +- gen/terraform/resource_type.go | 30 +- gen/terraform/tags.go | 5 - gen/util/util.go | 10 + go.sum | 1 + resource/generate.go | 3 +- resource/supported.go | 3 + 27 files changed, 838 insertions(+), 609 deletions(-) create mode 100644 aws/aws_cloudtrail.go create mode 100644 aws/aws_workspaces_directory.go rename aws/{aws_kms_alias.go => aws_workspaces_workspace.go} (50%) create mode 100644 gen/aws/README.tpl.md delete mode 100644 gen/aws/operation.go create mode 100644 gen/aws/util.go create mode 100644 gen/aws/util_test.go diff --git a/README.md b/README.md index 50f9be5..a4f49aa 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ A list command for AWS resources. [![Travis](https://img.shields.io/travis/jckuester/awsls/master.svg?style=for-the-badge)](https://travis-ci.org/jckuester/awsls) awsls supports listing of [over 200 types of resources](#supported-resources) -across 83 different AWS services. The goal is to code-generate a list function for +across 84 different AWS services. The goal is to code-generate a list function for every AWS resource that is covered by the Terraform AWS Provider (currently over 500). If you want to contribute, [the generator is here](./gen). @@ -96,7 +96,7 @@ The `--all-profiles` flag will use all profiles from `~/.aws/config`, or if `AWS ## Supported resources -Currently, all 234 resource types across 83 services in the table below can be listed with awsls. The `Tags` column shows if a resource +Currently, all 237 resource types across 84 services in the table below can be listed with awsls. The `Tags` column shows if a resource supports displaying tags, the `Creation Time` column if a resource has a creation timestamp, and the `Owner` column if resources are pre-filtered belonging to the account owner. @@ -141,6 +141,8 @@ Note: the prefix `aws_` for resource types is now optional. This means, for exam | aws_cloudformation_stack_set | x | | | **cloudhsmv2** | | aws_cloudhsm_v2_cluster | x | | +| **cloudtrail** | +| aws_cloudtrail | x | | | **cloudwatch** | | aws_cloudwatch_dashboard | | | | **cloudwatchevents** | @@ -209,7 +211,7 @@ Note: the prefix `aws_` for resource types is now optional. This means, for exam | aws_ec2_transit_gateway_vpc_attachment | x | x | | aws_egress_only_internet_gateway | x | | | aws_eip | x | | -| aws_instance | x | x | x | +| aws_instance | x | x | x | | aws_internet_gateway | x | | x | | aws_key_pair | x | | | aws_launch_template | x | x | @@ -419,6 +421,8 @@ Note: the prefix `aws_` for resource types is now optional. This means, for exam | **worklink** | | aws_worklink_fleet | | x | | **workspaces** | +| aws_workspaces_directory | x | | | aws_workspaces_ip_group | x | | +| aws_workspaces_workspace | x | | | **xray** | | aws_xray_group | x | | \ No newline at end of file diff --git a/aws/aws_cloudtrail.go b/aws/aws_cloudtrail.go new file mode 100644 index 0000000..6b0e462 --- /dev/null +++ b/aws/aws_cloudtrail.go @@ -0,0 +1,35 @@ +// Code is generated. DO NOT EDIT. + +package aws + +import ( + "context" + + "github.com/aws/aws-sdk-go-v2/service/cloudtrail" +) + +func ListCloudtrail(client *Client) ([]Resource, error) { + req := client.Cloudtrailconn.DescribeTrailsRequest(&cloudtrail.DescribeTrailsInput{}) + + var result []Resource + + resp, err := req.Send(context.Background()) + if err != nil { + return nil, err + } + + if len(resp.TrailList) > 0 { + for _, r := range resp.TrailList { + + result = append(result, Resource{ + Type: "aws_cloudtrail", + ID: *r.Name, + Profile: client.Profile, + Region: client.Region, + AccountID: client.AccountID, + }) + } + } + + return result, nil +} diff --git a/aws/aws_instance.go b/aws/aws_instance.go index 85239a1..ae749a9 100644 --- a/aws/aws_instance.go +++ b/aws/aws_instance.go @@ -1,5 +1,5 @@ -// TODO this resource is currently manually added, because the pattern of how -// it's code must be generated differs from all other resources (nested for-loop +// This resource is currently manually added, because the pattern of how +// it's code must be generated differs from other resources (nested for-loop // of instances inside reservations) package aws diff --git a/aws/aws_workspaces_directory.go b/aws/aws_workspaces_directory.go new file mode 100644 index 0000000..278cedd --- /dev/null +++ b/aws/aws_workspaces_directory.go @@ -0,0 +1,37 @@ +// Code is generated. DO NOT EDIT. + +package aws + +import ( + "context" + + "github.com/aws/aws-sdk-go-v2/service/workspaces" +) + +func ListWorkspacesDirectory(client *Client) ([]Resource, error) { + req := client.Workspacesconn.DescribeWorkspaceDirectoriesRequest(&workspaces.DescribeWorkspaceDirectoriesInput{}) + + var result []Resource + + p := workspaces.NewDescribeWorkspaceDirectoriesPaginator(req) + for p.Next(context.Background()) { + page := p.CurrentPage() + + for _, r := range page.Directories { + + result = append(result, Resource{ + Type: "aws_workspaces_directory", + ID: *r.DirectoryId, + Profile: client.Profile, + Region: client.Region, + AccountID: client.AccountID, + }) + } + } + + if err := p.Err(); err != nil { + return nil, err + } + + return result, nil +} diff --git a/aws/aws_kms_alias.go b/aws/aws_workspaces_workspace.go similarity index 50% rename from aws/aws_kms_alias.go rename to aws/aws_workspaces_workspace.go index 941fd20..bd6a261 100644 --- a/aws/aws_kms_alias.go +++ b/aws/aws_workspaces_workspace.go @@ -5,23 +5,23 @@ package aws import ( "context" - "github.com/aws/aws-sdk-go-v2/service/kms" + "github.com/aws/aws-sdk-go-v2/service/workspaces" ) -func ListKmsAlias(client *Client) ([]Resource, error) { - req := client.Kmsconn.ListAliasesRequest(&kms.ListAliasesInput{}) +func ListWorkspacesWorkspace(client *Client) ([]Resource, error) { + req := client.Workspacesconn.DescribeWorkspacesRequest(&workspaces.DescribeWorkspacesInput{}) var result []Resource - p := kms.NewListAliasesPaginator(req) + p := workspaces.NewDescribeWorkspacesPaginator(req) for p.Next(context.Background()) { page := p.CurrentPage() - for _, r := range page.Aliases { + for _, r := range page.Workspaces { result = append(result, Resource{ - Type: "aws_kms_alias", - ID: *r.AliasName, + Type: "aws_workspaces_workspace", + ID: *r.WorkspaceId, Profile: client.Profile, Region: client.Region, AccountID: client.AccountID, diff --git a/aws/client.go b/aws/client.go index a3479d4..911bc11 100644 --- a/aws/client.go +++ b/aws/client.go @@ -5,6 +5,7 @@ package aws import ( "context" "fmt" + "github.com/apex/log" "github.com/aws/aws-sdk-go-v2/aws/external" "github.com/aws/aws-sdk-go-v2/service/accessanalyzer" diff --git a/aws/list.go b/aws/list.go index d8308f4..9642f64 100644 --- a/aws/list.go +++ b/aws/list.go @@ -72,6 +72,8 @@ func ListResourcesByType(client *Client, resourceType string) ([]Resource, error return ListCloudformationStackSet(client) case "aws_cloudhsm_v2_cluster": return ListCloudhsmV2Cluster(client) + case "aws_cloudtrail": + return ListCloudtrail(client) case "aws_cloudwatch_dashboard": return ListCloudwatchDashboard(client) case "aws_cloudwatch_event_bus": @@ -486,8 +488,12 @@ func ListResourcesByType(client *Client, resourceType string) ([]Resource, error return ListWafv2WebAclLoggingConfiguration(client) case "aws_worklink_fleet": return ListWorklinkFleet(client) + case "aws_workspaces_directory": + return ListWorkspacesDirectory(client) case "aws_workspaces_ip_group": return ListWorkspacesIpGroup(client) + case "aws_workspaces_workspace": + return ListWorkspacesWorkspace(client) case "aws_xray_group": return ListXrayGroup(client) default: diff --git a/gen/aws/README.tpl.md b/gen/aws/README.tpl.md new file mode 100644 index 0000000..ee19eee --- /dev/null +++ b/gen/aws/README.tpl.md @@ -0,0 +1,112 @@ +# awsls + +A list command for AWS resources. + +[![Release](https://img.shields.io/github/release/jckuester/awsls.svg?style=for-the-badge)](https://github.com/jckuester/awsls/releases/latest) +[![Software License](https://img.shields.io/badge/license-MIT-brightgreen.svg?style=for-the-badge)](/LICENSE.md) +[![Travis](https://img.shields.io/travis/jckuester/awsls/master.svg?style=for-the-badge)](https://travis-ci.org/jckuester/awsls) + +awsls supports listing of [over 200 types of resources](#supported-resources) +across {{ len .Services }} different AWS services. The goal is to code-generate a list function for +every AWS resource that is covered by the Terraform AWS Provider (currently over 500). If you want to contribute, +[the generator is here](./gen). + +If you encounter any issue with `awsls` or have a feature request, +please open an issue or write me on [Twitter](https://twitter.com/jckuester). + +Happy listing! + +**Note:** If you're also looking for an easy but powerful way to delete AWS resources, pipe the output of `awsls` into its new sibling +[`awsrm`](https://github.com/jckuester/awsrm) via Unix-pipes and use well-known standard tooling such as `grep` for filtering. + +## Features + +* List multiple types of resources at once by using glob patterns + (e.g., `"aws_iam_*"` lists all IAM resources and `"*"` all resources in your account) +* **New:** List resources across multiple accounts and regions by using the `--profiles` and `--regions` flag + (e.g., `-p account1,account2 -r us-east-1,us-west-2`) +* Show any resource attribute documented in the [Terraform AWS Provider](https://registry.terraform.io/providers/hashicorp/aws/latest/docs) + (e.g., `-a private_ip,tags` lists the IP and tags for resources of type [`aws_instance`](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/instance#attributes-reference)) + +## Examples + +### List various resource attributes + +Use Terraform resource types to tell `awsls` which resources to list. For example, `awsls aws_instance` shows +all EC2 instances. In addition to the default attributes `TYPE`, `ID`, `REGION`, and `CREATED` timestamp, additional attributes +can be displayed via the `--attributes ` flag. Every attribute in the Terraform documentation +([here](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/instance#attributes-reference) are the attributes for `aws_instance`) is a valid one: + +![](img/instance.gif) + +### List multiple resource types at once (via glob patterns) + +For example, `awsls "aws_iam_*` lists all IAM resources: + +![](img/iam.gif) + +### List across multiple accounts and regions + +To use specific profiles and/or regions, use the `-p (--profiles)` or `-r (--regions)` flags. For example, +`-p myaccount1,myaccount2 -r us-east-1,us-west-2` lists resources in every permutation of the given profiles and regions, +i.e., resources in region `us-west-2` and `us-east-1` for account `myaccount1` as well as `myaccount2`: + +![](img/multi-profiles-and-regions.gif) + +## Usage + + awsls [flags] + +To see options available run `awsls --help`. + +## Installation + +### Binary Releases + +You can download a specific version of awsls on the [releases page](https://github.com/jckuester/awsls/releases) or +install it the following way to `./bin/`: + +```bash +curl -sSfL https://raw.githubusercontent.com/jckuester/awsls/master/install.sh | sh -s v0.6.1 +``` + +### Homebrew + +Homebrew users can install by: + +```bash +brew install jckuester/tap/awsls +``` + +For more information on Homebrew taps please see the [tap documentation](https://docs.brew.sh/Taps). + +## Credentials, profiles and regions + +If the `--profiles` and/or `--regions` flag is unset, `awsls` will follow the usual +[AWS CLI precedence](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-precedence) +of first trying to find credentials, profiles and/or regions via [environment variables](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html), +and so on. + +For example, if using `--profiles foo,bar`, but not setting the regions flag, +`awsls` will first try to use the region from an environment variable (e.g., `AWS_DEFAULT_REGION`) +and second will try to use the default region for each profile from `~/.aws/config`. + +The `--all-profiles` flag will use all profiles from `~/.aws/config`, or if `AWS_CONFIG_FILE=/my/config` is set, from +`/my/config` otherwise. + +## Supported resources + +Currently, all {{ .SupportedResourceTypeCount }} resource types across {{ len .Services }} services in the table below can be listed with awsls. The `Tags` column shows if a resource +supports displaying tags, the `Creation Time` column if a resource has a creation timestamp, and the `Owner` column if +resources are pre-filtered belonging to the account owner. + +Note: the prefix `aws_` for resource types is now optional. This means, for example, +`awsls aws_instance` and `awsls instance` are both valid commands. + +| Service / Type | Tags | Creation Time | Owner +| :------------- | :--: | :-----------: | :---: +{{ range $service := .Services }}| **{{ $service.Name }}** | +{{ range $rType := $service.TerraformResourceTypes -}} +| {{ $rType.Name }} | {{ if $rType.Tags }} x {{ end }} | {{ if $rType.CreationTime }} x {{ end }} |{{ if $rType.Owner }} x |{{ end }} +{{ end }} +{{- end }} \ No newline at end of file diff --git a/gen/aws/client.go b/gen/aws/client.go index 7c5296f..e00a5aa 100644 --- a/gen/aws/client.go +++ b/gen/aws/client.go @@ -4,8 +4,6 @@ package aws import ( "bytes" - "fmt" - "os" "path/filepath" "strings" "text/template" @@ -13,14 +11,9 @@ import ( "github.com/jckuester/awsls/gen/util" ) -// GenerateClient writes Go code to initialize all AWS API Clients. -func GenerateClient(outputPath string, services []string) error { - err := os.MkdirAll(outputPath, 0775) - if err != nil { - return fmt.Errorf("failed to create directory: %s", err) - } - - err = util.WriteGoFile( +// GenerateClient returns Go code that initializes all AWS API clients. +func GenerateClient(outputPath string, services []string) { + err := util.WriteGoFile( filepath.Join(outputPath, "client.go"), util.CodeLayout, "", @@ -29,10 +22,8 @@ func GenerateClient(outputPath string, services []string) error { ) if err != nil { - return fmt.Errorf("failed to write Go code to file: %s", err) + panic(err) } - - return nil } func clientGoCode(services []string) string { diff --git a/gen/aws/list.go b/gen/aws/list.go index 26a9cd2..a3f5798 100644 --- a/gen/aws/list.go +++ b/gen/aws/list.go @@ -3,170 +3,90 @@ package aws import ( + "bytes" "fmt" "path/filepath" - "sort" "strings" + "text/template" "github.com/apex/log" "github.com/aws/aws-sdk-go-v2/private/model/api" "github.com/jckuester/awsls/gen/util" ) -type GeneratedResourceInfo struct { - Type string +type Service struct { + Name string + TerraformResourceTypes []ResourceType +} + +type ResourceType struct { + Name string Tags bool CreationTime bool Owner bool } -func GenerateListFunctions(outputPath string, resourceServices map[string]string, resourceIDs map[string]string, - resourceTypesWithTags []string, apis api.APIs) (map[string]string, map[string][]GeneratedResourceInfo) { - listFunctionNames := map[string]string{} - genResourceInfo := map[string][]GeneratedResourceInfo{} +func GenerateListFunctions(outputPath string, services []Service, resourceIDs map[string]string, + resourceTypesWithTags []string, apis api.APIs) []Service { - for _, service := range ServicesCoveredByTerraform(resourceServices) { - fmt.Printf("\nservice: %s\n---\n", service) + resourcesWithRequiredFieldsCount := 0 + noOutputFieldNameFoundCount := 0 + noListOpCandidatesFoundCount := 0 + noResourceIDFoundCount := 0 - var genResourceInfoPerService []GeneratedResourceInfo + var servicesResult []Service - for rType, rService := range resourceServices { - if rService != service { - continue - } + for _, service := range services { + fmt.Println() + fmt.Printf("service: %s\n---\n", service.Name) - _, ok := ExcludedResourceTypes[rType] + var rTypesResult []ResourceType + + for _, rType := range service.TerraformResourceTypes { + _, ok := ExcludedResourceTypes[rType.Name] if ok { log.WithField("resource", rType).Info("exclude") continue } - service, ok := resourceServices[rType] - if !ok { - log.WithField("resource", rType).Warnf("service not found") - continue - } - - var op Operation - var outputFieldName string - - listOpCandidates := GetListOperationCandidates(rType, service, apis) + listOpCandidates := FindListOperationCandidates(rType.Name, service.Name, apis) if len(listOpCandidates) == 0 { - log.WithField("resource", rType).Warnf("no list operation candidate found") + noListOpCandidatesFoundCount++ + log.WithField("resource", rType).Errorf("no list operation candidate found") + continue } - var listOpCandidateWithFoundOutputField []string - for _, listOpCandidate := range listOpCandidates { - outputFieldCandidates := GetOutputFieldCandidates(rType, listOpCandidate) - if len(outputFieldCandidates) == 0 { - continue - } + outputFieldName, op, err := findOutputField(rType.Name, listOpCandidates, "structure") + if err != nil { + _, _, err = findOutputField(rType.Name, listOpCandidates, "string") + if err != nil { + noOutputFieldNameFoundCount++ + log.WithError(err).WithField("resource", rType.Name).Errorf("unable to find output field name") - if len(outputFieldCandidates) > 1 { - log.WithFields(log.Fields{ - "resource": rType, - "operation": listOpCandidate.ExportedName, - "candidates": outputFieldCandidates, - }).Infof("multiple output field candidates") continue } - listOpCandidateWithFoundOutputField = append(listOpCandidateWithFoundOutputField, listOpCandidate.ExportedName) - op = listOpCandidate - outputFieldName = outputFieldCandidates[0] - op.OutputListName = outputFieldName - } - - if len(listOpCandidateWithFoundOutputField) == 0 { - log.WithField("resource", rType).Warnf("no list operation candidate with struct found") - continue - } - - if len(listOpCandidateWithFoundOutputField) > 1 { - log.WithFields(log.Fields{ - "resource": rType, - "candidates": listOpCandidateWithFoundOutputField}).Warnf("multiple list operation candidates found") - continue - } + log.WithField("resource", rType.Name).Infof("found output field of type string") - if len(op.InputRef.Shape.Required) > 0 { - log.WithField("resource", rType).Warnf("required input fields: %s", op.InputRef.Shape.Required) continue } outputField := op.OutputRef.Shape.MemberRefs[outputFieldName] - resourceID, ok := ManualMatchedResourceID[rType] - if !ok { - resourceID, ok = resourceIDs[rType] - if !ok { - log.WithField("resource", rType).Warn("no ID found") - continue - } - - if resourceID == "NAME_PLACEHOLDER" { - resourceIDCandidates := GetResourceIDNameCandidates(outputField) - if len(resourceIDCandidates) > 1 { - log.WithFields(log.Fields{ - "resource": rType, - "candidates": resourceIDCandidates, - }).Warnf("found multiple name field ID candidates as resource ID for NAME_PLACEHOLDER") - - continue - } - - if len(resourceIDCandidates) == 0 { - log.WithFields(log.Fields{ - "resource": rType, - }).Warnf("found no name field candidates as resource ID for NAME_PLACEHOLDER") - - continue - } - - resourceID = resourceIDCandidates[0] - } - } - - op.TerraformType = rType - op.ResourceID = resourceID - op.OpName = TypeToOpName(rType) - - listFunctionNames[rType] = TypeToOpName(rType) - - genInfo := GeneratedResourceInfo{ - Type: rType, - } - - op.Inputs = Inputs[rType] - - op.GetTagsGoCode = GetTagsGoCode(outputField) - - /* - if op.GetTagsGoCode != "" { - genInfo.Tags = true - } - */ - - for _, rWithTags := range resourceTypesWithTags { - if rWithTags == rType { - genInfo.Tags = true - } - } - - getTagCode, imports := GetCreationTimeGoCode(outputField) - - op.GetCreationTimeGoCode = getTagCode - op.Imports = imports + if len(op.InputRef.Shape.Required) > 0 { + resourcesWithRequiredFieldsCount++ + log.WithField("resource", rType).Errorf("required input fields: %s", op.InputRef.Shape.Required) - if op.GetCreationTimeGoCode != "" { - genInfo.CreationTime = true + continue } - op.GetOwnerGoCode = GetOwnerGoCode(outputField) + resourceID, err := findResourceID(rType.Name, resourceIDs, outputField) + if err != nil { + noResourceIDFoundCount++ + log.WithField("resource", rType).Errorf("no resource ID found") - if op.GetOwnerGoCode != "" { - genInfo.Owner = true + continue } for k, _ := range op.InputRef.Shape.MemberRefs { @@ -175,66 +95,80 @@ func GenerateListFunctions(outputPath string, resourceServices map[string]string } } - genResourceInfoPerService = append(genResourceInfoPerService, genInfo) - - err := writeListFunction(outputPath, &op, rType) - if err != nil { - log.Fatal(err.Error()) + op.OutputFieldName = outputFieldName + op.TerraformType = rType.Name + op.ResourceID = resourceID + op.OpName = rType.ListFunctionName() + op.Inputs = Inputs[rType.Name] + + rType.CreationTime = op.GetCreationTimeGoCode() != "" + rType.Owner = op.GetOwnerGoCode() != "" + rType.Tags = util.Contains(resourceTypesWithTags, rType.Name) + + if rType.Name != "aws_instance" { + // note: code is manually added for "aws_instance" + writeListFunction(outputPath, &op) + } else { + rType.CreationTime = true } + + rTypesResult = append(rTypesResult, rType) } - if len(genResourceInfoPerService) > 0 { - sort.Slice(genResourceInfoPerService, func(i, j int) bool { - return genResourceInfoPerService[i].Type < genResourceInfoPerService[j].Type + if len(rTypesResult) > 0 { + servicesResult = append(servicesResult, Service{ + Name: service.Name, + TerraformResourceTypes: rTypesResult, }) - - genResourceInfo[service] = genResourceInfoPerService } } - return listFunctionNames, genResourceInfo -} + log.Infof("list functions with required fields: %d", resourcesWithRequiredFieldsCount) + log.Infof("unable to find output field name: %d", noOutputFieldNameFoundCount) + log.Infof("resources without list operation candidate: %d", noListOpCandidatesFoundCount) + log.Infof("no resource ID found: %d", noResourceIDFoundCount) -func GetResourceIDNameCandidates(v *api.ShapeRef) []string { - var result []string + return servicesResult +} - for k, _ := range v.Shape.MemberRef.Shape.MemberRefs { - if k == "Name" { - return []string{k} - } +func writeListFunction(outputPath string, op *ListOperation) { + err := util.WriteGoFile( + filepath.Join(outputPath, op.TerraformType+".go"), + util.CodeLayout, + "", + "aws", + op.GoCode(), + ) - if strings.Contains(strings.ToLower(k), "name") { - result = append(result, k) - } + if err != nil { + panic(err) } - - return result } -// GetOutputFieldCandidates gets the output field that contains a list of resources the given resource type -// (e.g., field name LogGroups of type []LogGroup in output DescribeLogGroupsOutput) -// -// Note: if there is a manual match entry, this will be returned. -func GetOutputFieldCandidates(resourceType string, op Operation) []string { - _, ok := ManualMatchedOutputFields[resourceType] - if ok { - return []string{ManualMatchedOutputFields[resourceType]} - } +type ListOperation struct { + api.Operation - var outputFieldCandidates []string + TerraformType string + ResourceID string + OutputListName string + OpName string + Inputs string + OutputFieldName string +} - for fieldName, v := range op.OutputRef.Shape.MemberRefs { - if v.Shape.Type == "list" { - if v.Shape.MemberRef.Shape.Type == "structure" { - outputFieldCandidates = append(outputFieldCandidates, fieldName) - } - } +func (o *ListOperation) GoCode() string { + var buf bytes.Buffer + err := listResourcesOperationTmpl.Execute(&buf, o) + if err != nil { + panic(err) } - return outputFieldCandidates + return strings.TrimSpace(buf.String()) } -func GetTagsGoCode(outputField *api.ShapeRef) string { +func (o ListOperation) GetTagsGoCode() string { + outputField := o.OutputRef.Shape.MemberRefs[o.OutputFieldName] + for k, v := range outputField.Shape.MemberRef.Shape.MemberRefs { if k == "Tags" { if v.Shape.Type == "list" { @@ -260,7 +194,9 @@ func GetTagsGoCode(outputField *api.ShapeRef) string { return "" } -func GetCreationTimeGoCode(outputField *api.ShapeRef) (string, []string) { +func (o ListOperation) GetCreationTimeGoCode() string { + outputField := o.OutputRef.Shape.MemberRefs[o.OutputFieldName] + creationTimeFieldNames := []string{ "LaunchTime", "CreateTime", @@ -280,15 +216,15 @@ func GetCreationTimeGoCode(outputField *api.ShapeRef) (string, []string) { return `t, err := time.Parse("2006-01-02T15:04:05.000Z0700", *r.` + k + `) if err != nil { return nil, err - }`, []string{"time"} + }` } if v.Shape.Type == "timestamp" { - return `t := ` + fmt.Sprintf("*r.%s", k), []string{} + return `t := ` + fmt.Sprintf("*r.%s", k) } if v.Shape.Type == "long" { - return fmt.Sprintf("t := time.Unix(0, *r.%s * 1000000).UTC()", k), []string{"time"} + return fmt.Sprintf("t := time.Unix(0, *r.%s * 1000000).UTC()", k) } log.Warnf("uncovered creation time type: %s", v.Shape.Type) @@ -296,10 +232,12 @@ func GetCreationTimeGoCode(outputField *api.ShapeRef) (string, []string) { } } - return "", []string{} + return "" } -func GetOwnerGoCode(outputField *api.ShapeRef) string { +func (o ListOperation) GetOwnerGoCode() string { + outputField := o.OutputRef.Shape.MemberRefs[o.OutputFieldName] + for k, _ := range outputField.Shape.MemberRef.Shape.MemberRefs { if k == "OwnerId" { return `if *r.OwnerId != client.AccountID { @@ -314,133 +252,72 @@ func GetOwnerGoCode(outputField *api.ShapeRef) string { return "" } -func Operations(apis api.APIs, prefixes []string) []string { - var result []string - - for _, a := range apis { - for _, v := range a.Operations { - for _, prefix := range prefixes { - if strings.HasPrefix(v.Name, prefix) && !strings.Contains(v.Name, "Tags") { - log.Debugf("%s", v.Name) - result = append(result, v.Name) - } - } - } - } - return result -} - -type ListOperationCandidates struct { - List *api.Operation - Get *api.Operation - Describes *api.Operation -} +var listResourcesOperationTmpl = template.Must(template.New("listResourcesOperation").Funcs( + template.FuncMap{ + "Title": strings.Title, + }).Parse(` +import( + "context" + "github.com/aws/aws-sdk-go-v2/service/{{ .API.PackageName }}" +) -// GetListOperationCandidates gets the possible list operation -// -// Note: -// * The list operation can be a Get, List or Describe function -// * If there is a manual match entry, this will be returned. -func GetListOperationCandidates(resourceType, service string, apis api.APIs) []Operation { - manualMatchedOp, ok := ManualMatchedListOps[resourceType] - if ok { - for _, op := range operationsOfService(apis, service, "") { - if op.ExportedName == manualMatchedOp { - return []Operation{{Operation: *op}} - } +{{ $reqType := printf "%sRequest" .ExportedName -}} +{{ $pagerType := printf "%sPaginator" .ExportedName -}} + +func {{.OpName}}(client *Client) ([]Resource, error) { + req := client.{{ .API.PackageName | Title }}conn.{{ $reqType }}(&{{ .API.PackageName }}.{{ .InputRef.GoTypeElem }}{ {{ if ne .Inputs "" }}{{ .Inputs }}{{ end }} }) + + var result []Resource + + {{ if .Paginator }} + p := {{ .API.PackageName }}.New{{ $pagerType }}(req) + for p.Next(context.Background()) { + page := p.CurrentPage() + + for _, r := range page.{{ .OutputListName }}{ + {{ if ne .GetOwnerGoCode "" }}{{ .GetOwnerGoCode }}{{ end }} + {{ if ne .GetTagsGoCode "" }}{{ .GetTagsGoCode }}{{ end }} + {{ if ne .GetCreationTimeGoCode "" }}{{ .GetCreationTimeGoCode }}{{ end }} + result = append(result, Resource{ + Type: "{{ .TerraformType }}", + ID: *r.{{ .ResourceID }}, + Profile: client.Profile, + Region: client.Region, + AccountID: client.AccountID, + {{ if ne .GetTagsGoCode "" }}Tags: tags,{{ end }} + {{ if ne .GetCreationTimeGoCode "" }}CreatedAt: &t,{{ end }} + }) } } - var result []Operation - - prefixes := []string{"Describe", "Get", "List"} - - for _, prefix := range prefixes { - operations := operationsOfService(apis, service, prefix) - - matchingOp := exactMatch(resourceType, operations, prefix) - if matchingOp != nil { - result = append(result, Operation{Operation: *matchingOp}) - } + if err := p.Err(); err != nil { + return nil, err } - return result -} - -func exactMatch(terraformType string, operations []*api.Operation, opPrefix string) *api.Operation { - plurals := pluralizeListFunctionCandidateNames(terraformType) - - for _, t := range plurals { - for _, op := range operations { - opNoPrefix := strings.ToLower(strings.TrimPrefix(op.ExportedName, opPrefix)) + {{ else }} - if t == opNoPrefix { - return op - } - } - } - - return nil -} - -func pluralizeListFunctionCandidateNames(terraformType string) []string { - tNoPrefix := strings.TrimPrefix(terraformType, "aws_") - tSplit := strings.Split(tNoPrefix, "_") - - var tCombined []string - for i := 0; i < len(tSplit); i++ { - tCombined = append(tCombined, strings.Join(tSplit[i:], "")) - } - - var plural []string - for _, c := range tCombined { - plural = append(plural, []string{c + "s", c + "es"}...) - if strings.HasSuffix(c, "y") { - plural = append(plural, strings.TrimSuffix(c, "y")+"ies") - } + resp, err := req.Send(context.Background()) + if err != nil { + return nil, err } - return plural -} - -// operationsOfService returns the operations with a given prefix that belong to a service. -func operationsOfService(apis api.APIs, service, opPrefix string) []*api.Operation { - var result []*api.Operation - - for _, api := range apis { - if service != api.PackageName() { - continue - } - for _, op := range api.Operations { - if strings.HasPrefix(op.ExportedName, opPrefix) { - result = append(result, op) - } + if len(resp.{{ .OutputListName }}) > 0 { + for _, r := range resp.{{ .OutputListName }}{ + {{ if ne .GetOwnerGoCode "" }}{{ .GetOwnerGoCode }}{{ end }} + {{ if ne .GetTagsGoCode "" }}{{ .GetTagsGoCode }}{{ end }} + {{ if ne .GetCreationTimeGoCode "" }}{{ .GetCreationTimeGoCode }}{{ end }} + result = append(result, Resource{ + Type: "{{ .TerraformType }}", + ID: *r.{{ .ResourceID }}, + Profile: client.Profile, + Region: client.Region, + AccountID: client.AccountID, + {{ if ne .GetTagsGoCode "" }}Tags: tags,{{ end }} + {{ if ne .GetCreationTimeGoCode "" }}CreatedAt: &t,{{ end }} + }) } } - - return result -} - -// TypeToOpName generates a name for the list function based on the resource type. -func TypeToOpName(terraformType string) string { - split := strings.Split(strings.TrimPrefix(terraformType, "aws_"), "_") - capitalized := strings.Title(strings.Join(split, " ")) - - return strings.Join(strings.Split(capitalized, " "), "") -} - -func writeListFunction(outputPath string, op *Operation, terraformType string) error { - err := util.WriteGoFile( - filepath.Join(outputPath, terraformType+".go"), - util.CodeLayout, - "", - "aws", - op.GoCode(), - ) - - if err != nil { - return fmt.Errorf("failed to write %s Go Code to file, %v", op.ExportedName, err) - } - - return nil + {{ end }} + return result, nil } +`)) diff --git a/gen/aws/manual_mapping.go b/gen/aws/manual_mapping.go index b3f8538..67ded8e 100644 --- a/gen/aws/manual_mapping.go +++ b/gen/aws/manual_mapping.go @@ -9,6 +9,8 @@ var excludeServices = map[string]struct{}{ // some resource types are excluded as they need be handled slightly differently var ExcludedResourceTypes = map[string]bool{ + // not a resource + "aws_api_gateway_integration": true, // is not a resource "aws_acm_certificate_validation": true, // ValidationError: You must specify either either listener ARNs or a load balancer ARN @@ -33,7 +35,6 @@ var ExcludedResourceTypes = map[string]bool{ "aws_inspector_assessment_target": true, "aws_inspector_assessment_template": true, "aws_inspector_resource_group": true, - "aws_instance": true, "aws_resourcegroups_group": true, "aws_shield_protection": true, "aws_vpn_connection": true, @@ -50,10 +51,15 @@ var ExcludedResourceTypes = map[string]bool{ "aws_default_vpc": true, // Error: failed to read current state of resource: rpc error: code = Canceled desc = context canceled "aws_kms_alias": true, + // Not a resource. This are the instances registered with an ELB. Requires the ELB name. + "aws_elb_attachment": true, } // manualMatchedListOps are list operations that could not be matched automatically var ManualMatchedListOps = map[string]string{ + "aws_appautoscaling_target": "DescribeScalableTargets", + "aws_appautoscaling_policy": "DescribeScalingPolicies", + "aws_appautoscaling_scheduled_action": "DescribeScheduledActions", // The AWS API calls it images not AMI "aws_ami": "DescribeImages", "aws_ecs_service": "DescribeServices", @@ -66,6 +72,8 @@ var ManualMatchedListOps = map[string]string{ "aws_ssm_parameter": "DescribeParameters", "aws_ssm_resource_data_sync": "ListResourceDataSync", "aws_lb": "DescribeLoadBalancers", + "aws_cloudtrail": "DescribeTrails", + "aws_workspaces_directory": "DescribeWorkspaceDirectories", } var ManualMatchedOutputFields = map[string]string{ @@ -74,16 +82,21 @@ var ManualMatchedOutputFields = map[string]string{ } var ManualMatchedResourceID = map[string]string{ - "aws_autoscaling_group": "AutoScalingGroupName", - "aws_launch_configuration": "LaunchConfigurationName", - "aws_s3_bucket": "Name", - "aws_elb": "LoadBalancerName", - "aws_db_instance": "DBInstanceIdentifier", - "aws_route53_zone": "Id", - "aws_subnet": "SubnetId", - "aws_imagebuilder_component": "Arn", + "aws_autoscaling_group": "AutoScalingGroupName", + "aws_launch_configuration": "LaunchConfigurationName", + "aws_s3_bucket": "Name", + "aws_elb": "LoadBalancerName", + "aws_db_instance": "DBInstanceIdentifier", + "aws_route53_zone": "Id", + "aws_ses_email_identity": "EmailAddress", + "aws_ses_domain_identity": "Domain", + "aws_simpledb_domain": "DomainName", + "aws_workspaces_workspace": "WorkspaceId", + "aws_subnet": "SubnetId", + "aws_imagebuilder_component": "Arn", "aws_imagebuilder_distribution_configuration": "Arn", "aws_imagebuilder_infrastructure_configuration": "Arn", + "aws_workspaces_directory": "DirectoryId", } var Inputs = map[string]string{ diff --git a/gen/aws/operation.go b/gen/aws/operation.go deleted file mode 100644 index 2e89bc1..0000000 --- a/gen/aws/operation.go +++ /dev/null @@ -1,108 +0,0 @@ -// +build codegen - -package aws - -import ( - "bytes" - "strings" - "text/template" - - "github.com/aws/aws-sdk-go-v2/private/model/api" -) - -type Operation struct { - api.Operation - - TerraformType string - ResourceID string - OutputListName string - OpName string - GetTagsGoCode string - GetCreationTimeGoCode string - GetOwnerGoCode string - Inputs string - Imports []string -} - -func (o *Operation) GoCode() string { - var buf bytes.Buffer - err := listResourcesOperationTmpl.Execute(&buf, o) - if err != nil { - panic(err) - } - - return strings.TrimSpace(buf.String()) -} - -var listResourcesOperationTmpl = template.Must(template.New("listResourcesOperation").Funcs( - template.FuncMap{ - "Title": strings.Title, - }).Parse(` -import( - "context" - {{ range .Imports }}"{{ . }}" - {{ end }} - "github.com/aws/aws-sdk-go-v2/service/{{ .API.PackageName }}" -) - -{{ $reqType := printf "%sRequest" .ExportedName -}} -{{ $respType := printf "%sResponse" .ExportedName -}} -{{ $pagerType := printf "%sPaginator" .ExportedName -}} - -func List{{.OpName}}(client *Client) ([]Resource, error) { - req := client.{{ .API.PackageName | Title }}conn.{{ $reqType }}(&{{ .API.PackageName }}.{{ .InputRef.GoTypeElem }}{ {{ if ne .Inputs "" }}{{ .Inputs }}{{ end }} }) - - var result []Resource - - {{ if .Paginator }} - p := {{ .API.PackageName }}.New{{ $pagerType }}(req) - for p.Next(context.Background()) { - page := p.CurrentPage() - - for _, r := range page.{{ .OutputListName }}{ - {{ if ne .GetOwnerGoCode "" }}{{ .GetOwnerGoCode }}{{ end }} - {{ if ne .GetTagsGoCode "" }}{{ .GetTagsGoCode }}{{ end }} - {{ if ne .GetCreationTimeGoCode "" }}{{ .GetCreationTimeGoCode }}{{ end }} - result = append(result, Resource{ - Type: "{{ .TerraformType }}", - ID: *r.{{ .ResourceID }}, - Profile: client.Profile, - Region: client.Region, - AccountID: client.AccountID, - {{ if ne .GetTagsGoCode "" }}Tags: tags,{{ end }} - {{ if ne .GetCreationTimeGoCode "" }}CreatedAt: &t,{{ end }} - }) - } - } - - if err := p.Err(); err != nil { - return nil, err - } - - {{ else }} - - resp, err := req.Send(context.Background()) - if err != nil { - return nil, err - } - - if len(resp.{{ .OutputListName }}) > 0 { - for _, r := range resp.{{ .OutputListName }}{ - {{ if ne .GetOwnerGoCode "" }}{{ .GetOwnerGoCode }}{{ end }} - {{ if ne .GetTagsGoCode "" }}{{ .GetTagsGoCode }}{{ end }} - {{ if ne .GetCreationTimeGoCode "" }}{{ .GetCreationTimeGoCode }}{{ end }} - result = append(result, Resource{ - Type: "{{ .TerraformType }}", - ID: *r.{{ .ResourceID }}, - Profile: client.Profile, - Region: client.Region, - AccountID: client.AccountID, - {{ if ne .GetTagsGoCode "" }}Tags: tags,{{ end }} - {{ if ne .GetCreationTimeGoCode "" }}CreatedAt: &t,{{ end }} - }) - } - } - {{ end }} - return result, nil -} -`)) diff --git a/gen/aws/readme.go b/gen/aws/readme.go index 6a5623d..f3bc8a8 100644 --- a/gen/aws/readme.go +++ b/gen/aws/readme.go @@ -4,73 +4,37 @@ package aws import ( "bytes" - "fmt" "io/ioutil" "path/filepath" - "sort" "strings" "text/template" ) type readmeData struct { - Services []string - ResourceInfos map[string][]GeneratedResourceInfo + Services []Service SupportedResourceTypeCount int } -func WriteReadme(outputPath string, resourceInfos map[string][]GeneratedResourceInfo) error { - err := ioutil.WriteFile(filepath.Join(outputPath, "README.md"), - []byte(readmeCode(resourceInfos)), 0664) - +func GenerateReadme(outputPath string, services []Service, numOfResourceTypes int) { + err := ioutil.WriteFile(filepath.Join(outputPath, "README.md"), []byte(readmeCode(services, numOfResourceTypes)), 0664) if err != nil { - return fmt.Errorf("failed to write README.md: %s", err) + panic(err) } - - return nil } -func readmeCode(resourceInfos map[string][]GeneratedResourceInfo) string { +func readmeCode(services []Service, numOfResourceTypes int) string { var buf bytes.Buffer - resourceCount := 0 - - var services []string - for service, resources := range resourceInfos { - resourceCount += len(resources) - services = append(services, service) - } - - sort.Strings(services) + tpl := template.Must(template.New("README.tpl.md").ParseFiles("../gen/aws/README.tpl.md")) - err := Readme.Execute(&buf, readmeData{ + err := tpl.Execute(&buf, readmeData{ Services: services, - ResourceInfos: resourceInfos, - SupportedResourceTypeCount: resourceCount, + SupportedResourceTypeCount: numOfResourceTypes, }) + if err != nil { panic(err) } return strings.TrimSpace(buf.String()) } - -var Readme = template.Must(template.New("readme").Parse(` -{{ $infos := .ResourceInfos }} - -# awsls - -## Supported Resource Types - -Currently, all {{ .SupportedResourceTypeCount }} resource types across {{ len .Services }} services in the table below can be -listed with awsls. The "Tags" column shows if a resource supports displaying tags, the "Creation Time" column if a -resource has a creation timestamp, and the "Owner" column if resources are pre-filtered by account ID. - -| Service / Type | Tags | Creation Time | Owner -| :------------- | :--: | :-----------: | :---: -{{ range .Services }}{{ $service := . }}| **{{ $service }}** | -{{ range $key, $value := $infos }}{{ if eq $key $service -}} -{{ range $value -}} -| {{ .Type }} | {{ if .Tags }} x {{ end }} | {{ if .CreationTime }} x {{ end }} |{{ if .Owner }} x |{{ end }} -{{ end }} -{{- end }}{{ end }}{{ end }} -`)) diff --git a/gen/aws/service.go b/gen/aws/service.go index 1c75843..a53954e 100644 --- a/gen/aws/service.go +++ b/gen/aws/service.go @@ -24,21 +24,35 @@ func ServicePkgNames(apis api.APIs) []string { return result } -// ServicesCoveredByTerraform returns the package name of all AWS services that are -// (at least partially) covered by Terraform resources. -func ServicesCoveredByTerraform(serviceMap map[string]string) []string { - serviceSet := make(map[string]bool) - - for _, v := range serviceMap { - serviceSet[v] = true +// ResourceTypesForAWSServices returns the Terraform resource types belonging to each AWS service. +// An AWS service is only part of the result if its associated resource type list is not empty. +func ResourceTypesForAWSServices(serviceMap map[string]string) []Service { + rTypesByService := make(map[string][]ResourceType) + + for rType, service := range serviceMap { + rTypes, ok := rTypesByService[service] + if !ok { + rTypesByService[service] = []ResourceType{{Name: rType}} + continue + } + rTypesByService[service] = append(rTypes, ResourceType{Name: rType}) } - var result []string - for k, _ := range serviceSet { - result = append(result, k) + var result []Service + for service, rTypes := range rTypesByService { + sort.Slice(rTypes, func(i, j int) bool { + return rTypes[i].Name < rTypes[j].Name + }) + + result = append(result, Service{ + Name: service, + TerraformResourceTypes: rTypes, + }) } - sort.Strings(result) + sort.Slice(result, func(i, j int) bool { + return result[i].Name < result[j].Name + }) return result } diff --git a/gen/aws/supported.go b/gen/aws/supported.go index cb3270c..1deb6b7 100644 --- a/gen/aws/supported.go +++ b/gen/aws/supported.go @@ -4,7 +4,6 @@ package aws import ( "bytes" - "fmt" "path/filepath" "strings" "text/template" @@ -12,27 +11,25 @@ import ( "github.com/jckuester/awsls/gen/util" ) -// GenerateSupportedResourceTypeList generates code of a list of Terraform resource types that are currently -// supported by awsls and writes the code to directory outputPath. -func GenerateSupportedResourceTypeList(outputPath string, listFunctionNames map[string]string) error { +// GenerateListOfSupportedResourceTypes code-generates a list of Terraform resource types +// that are currently supported by awsls. +func GenerateListOfSupportedResourceTypes(outputPath string, rTypes []ResourceType) { err := util.WriteGoFile( filepath.Join(outputPath, "supported.go"), util.CodeLayout, "", "resource", - supportedResourcesGoCode(listFunctionNames), + supportedResourcesGoCode(rTypes), ) if err != nil { - return fmt.Errorf("failed to write Go code to file: %s", err) + panic(err) } - - return nil } -func supportedResourcesGoCode(listFunctionNames map[string]string) string { +func supportedResourcesGoCode(rTypes []ResourceType) string { var buf bytes.Buffer - err := supportedResourcesTmpl.Execute(&buf, listFunctionNames) + err := supportedResourcesTmpl.Execute(&buf, rTypes) if err != nil { panic(err) } @@ -43,6 +40,8 @@ func supportedResourcesGoCode(listFunctionNames map[string]string) string { var supportedResourcesTmpl = template.Must(template.New("supportedResources").Parse(` // SupportedTypes is a list of all resource types currently supported by awsls. var SupportedTypes = []string{ -{{ range $key, $value := . }}"{{ $key }}", -{{ end }}} + {{- range . }} + "{{ .Name }}", + {{- end }} +} `)) diff --git a/gen/aws/util.go b/gen/aws/util.go new file mode 100644 index 0000000..20f16d8 --- /dev/null +++ b/gen/aws/util.go @@ -0,0 +1,244 @@ +// +build codegen + +package aws + +import ( + "fmt" + "sort" + "strings" + + "github.com/aws/aws-sdk-go-v2/private/model/api" + + "github.com/apex/log" +) + +// ListFunctionName generates a name for the list function of the resource type. +func (r ResourceType) ListFunctionName() string { + split := strings.Split(strings.TrimPrefix(r.Name, "aws_"), "_") + capitalize := strings.Title(strings.Join(split, " ")) + + return "List" + strings.Join(strings.Split(capitalize, " "), "") +} + +func findResourceID(rType string, resourceIDs map[string]string, outputField *api.ShapeRef) (string, error) { + resourceID, ok := ManualMatchedResourceID[rType] + if ok { + return resourceID, nil + } + + resourceID, ok = resourceIDs[rType] + if !ok { + return "", fmt.Errorf("no resource ID found") + } + + if resourceID == "NAME_PLACEHOLDER" { + resourceIDCandidates := GetResourceIDNameCandidates(outputField) + if len(resourceIDCandidates) > 1 { + return "", fmt.Errorf("found multiple name field ID candidates as resource ID for NAME_PLACEHOLDER") + + } + + if len(resourceIDCandidates) == 0 { + return "", fmt.Errorf("found no name field candidates as resource ID for NAME_PLACEHOLDER") + } + + resourceID = resourceIDCandidates[0] + } + + return resourceID, nil +} + +func GetResourceIDNameCandidates(v *api.ShapeRef) []string { + var result []string + + for k, _ := range v.Shape.MemberRef.Shape.MemberRefs { + if k == "Name" { + return []string{k} + } + + if strings.Contains(strings.ToLower(k), "name") { + result = append(result, k) + } + } + + return result +} + +// GetOutputFieldCandidates gets the output field that contains a list of resources of the given resource type +// (e.g., field name LogGroups of type []LogGroup in DescribeLogGroupsOutput) +// +// Note: if there is a manual match entry, this will be returned. +func GetOutputFieldCandidates(resourceType string, op ListOperation, shapeType string) []string { + _, ok := ManualMatchedOutputFields[resourceType] + if ok { + return []string{ManualMatchedOutputFields[resourceType]} + } + + var outputFieldCandidates []string + + for fieldName, v := range op.OutputRef.Shape.MemberRefs { + if v.Shape.Type == "list" { + if v.Shape.MemberRef.Shape.Type == shapeType { + outputFieldCandidates = append(outputFieldCandidates, fieldName) + } + } + } + + return outputFieldCandidates +} + +func findOutputField(rType string, listOpCandidates []ListOperation, shapeType string) (string, ListOperation, error) { + var listOpCandidatesWithFoundOutputField []string + var outputFieldName string + var op ListOperation + + for _, listOpCandidate := range listOpCandidates { + outputFieldCandidates := GetOutputFieldCandidates(rType, listOpCandidate, shapeType) + if len(outputFieldCandidates) == 0 { + continue + } + + if len(outputFieldCandidates) > 1 { + log.WithFields(log.Fields{ + "resource": rType, + "operation": listOpCandidate.ExportedName, + "candidates": outputFieldCandidates, + }).Warnf("multiple output field candidates") + continue + } + + listOpCandidatesWithFoundOutputField = append(listOpCandidatesWithFoundOutputField, listOpCandidate.ExportedName) + op = listOpCandidate + outputFieldName = outputFieldCandidates[0] + op.OutputListName = outputFieldName + } + + if len(listOpCandidatesWithFoundOutputField) == 0 { + return "", op, fmt.Errorf("no list operation candidate with struct found") + } + + if len(listOpCandidatesWithFoundOutputField) > 1 { + return "", op, fmt.Errorf("multiple list operation candidates found: %s", listOpCandidatesWithFoundOutputField) + } + + return outputFieldName, op, nil +} + +func Operations(apis api.APIs, prefixes []string) []string { + var result []string + + for _, a := range apis { + for _, v := range a.Operations { + for _, prefix := range prefixes { + if strings.HasPrefix(v.Name, prefix) && !strings.Contains(v.Name, "Tags") { + log.Debugf("%s", v.Name) + result = append(result, v.Name) + } + } + } + } + return result +} + +type ListOperationCandidates struct { + List *api.Operation + Get *api.Operation + Describes *api.Operation +} + +// FindListOperationCandidates returns all list operation candidates for a resource type. +// A list operation is a candidate, if +// * it's name starts with Get, List or Describe +// * the operation belongs to the same service as the resource type +// * the name of the operation is a plural of the resource type name +// Note: If there is a manual match entry, this will be returned. +func FindListOperationCandidates(resourceType, service string, apis api.APIs) []ListOperation { + manualMatchedOp, ok := ManualMatchedListOps[resourceType] + if ok { + for _, op := range operationsOfService(apis, service, "") { + if op.ExportedName == manualMatchedOp { + return []ListOperation{{Operation: *op}} + } + } + } + + var result []ListOperation + + prefixes := []string{"Describe", "Get", "List"} + var ops []string + + for _, prefix := range prefixes { + operations := operationsOfService(apis, service, prefix) + + for _, op := range operations { + ops = append(ops, op.ExportedName) + } + sort.Strings(ops) + + matchingOp := exactMatch(resourceType, operations, prefix) + if matchingOp != nil { + result = append(result, ListOperation{Operation: *matchingOp}) + } + } + + if len(result) == 0 { + log.Debugf("list operations: %s", ops) + } + + return result +} + +func exactMatch(rType string, operations []*api.Operation, opPrefix string) *api.Operation { + plurals := pluralizeListFunctionCandidateNames(rType) + + for _, plural := range plurals { + for _, op := range operations { + opWithoutPrefix := strings.ToLower(strings.TrimPrefix(op.ExportedName, opPrefix)) + + if plural == opWithoutPrefix { + return op + } + } + } + + return nil +} + +func pluralizeListFunctionCandidateNames(rType string) []string { + rTypeWithoutPrefix := strings.TrimPrefix(rType, "aws_") + tSplit := strings.Split(rTypeWithoutPrefix, "_") + + var tCombined []string + for i := 0; i < len(tSplit); i++ { + tCombined = append(tCombined, strings.Join(tSplit[i:], "")) + } + + var result []string + for _, c := range tCombined { + result = append(result, []string{c + "s", c + "es"}...) + if strings.HasSuffix(c, "y") { + result = append(result, strings.TrimSuffix(c, "y")+"ies") + } + } + + return result +} + +// operationsOfService returns the operations with a given prefix that belong to a service. +func operationsOfService(apis api.APIs, service, opPrefix string) []*api.Operation { + var result []*api.Operation + + for _, api := range apis { + if service != api.PackageName() { + continue + } + + for _, op := range api.Operations { + if strings.HasPrefix(op.ExportedName, opPrefix) { + result = append(result, op) + } + } + } + + return result +} diff --git a/gen/aws/util_test.go b/gen/aws/util_test.go new file mode 100644 index 0000000..d48cf78 --- /dev/null +++ b/gen/aws/util_test.go @@ -0,0 +1,29 @@ +// +build codegen + +package aws + +import "testing" + +func TestListFunctionName(t *testing.T) { + tests := []struct { + name string + rType ResourceType + want string + }{ + { + name: "aws_iam_role", + rType: ResourceType{ + Name: "aws_iam_role", + }, + want: "ListIamRole", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.rType.ListFunctionName(); got != tt.want { + t.Errorf("ListFunctionName() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/gen/main.go b/gen/main.go index 81c48a7..0a22279 100644 --- a/gen/main.go +++ b/gen/main.go @@ -3,15 +3,20 @@ package main import ( + "io/ioutil" + stdlog "log" + "os" + "sort" + "github.com/apex/log" "github.com/apex/log/handlers/cli" "github.com/jckuester/awsls/gen/aws" "github.com/jckuester/awsls/gen/terraform" - "github.com/jckuester/awsls/gen/util" ) const ( - outputPath = "../resource" + outputPathAWS = "../aws" + outputPathResource = "../resource" terraformAwsProviderRepoPath = "/home/jan/git/github.com/terraform-provider-aws" awsSdkGoRepoPath = "/home/jan/git/github.com/aws-sdk-go-v2" ) @@ -20,7 +25,43 @@ func main() { log.SetHandler(cli.Default) log.SetLevel(log.DebugLevel) - resourceTypes, err := terraform.GenerateResourceTypeList(terraformAwsProviderRepoPath, outputPath) + // discard TRACE logs of GRPCProvider + stdlog.SetOutput(ioutil.Discard) + + profile := "myaccount" + region := "us-west-2" + + _, ok := os.LookupEnv("AWS_PROFILE") + if !ok { + log.Infof("AWS_PROFILE not set, therefore using the following default value: %s", profile) + + err := os.Setenv("AWS_PROFILE", profile) + if err != nil { + log.Fatal(err.Error()) + } + } + + _, ok = os.LookupEnv("AWS_DEFAULT_REGION") + if !ok { + log.Infof("AWS_DEFAULT_REGION not set, therefore using the following default value: %s", region) + + err := os.Setenv("AWS_DEFAULT_REGION", region) + if err != nil { + log.Fatal(err.Error()) + } + } + + err := os.MkdirAll(outputPathResource, 0775) + if err != nil { + log.Fatalf("failed to create directory: %s", err) + } + + err = os.MkdirAll(outputPathAWS, 0775) + if err != nil { + log.Fatalf("failed to create directory: %s", err) + } + + resourceTypes, err := terraform.GenerateResourceTypeList(terraformAwsProviderRepoPath, outputPathResource) if err != nil { log.WithError(err).Fatal("failed to generate list of Terraform AWS resource types") } @@ -30,18 +71,15 @@ func main() { log.WithError(err).Fatal("failed to generate map of resource type -> filename implementing resource") } - resourceServices, err := terraform.GenerateResourceServiceMap(terraformAwsProviderRepoPath, outputPath, + resourceServices := terraform.GenerateResourceServiceMap(terraformAwsProviderRepoPath, outputPathResource, resourceTypes, resourceFileNames) - if err != nil { - log.WithError(err).Fatal("failed to generate map of resource type -> AWS service") - } - resourceIDs, err := terraform.GenerateResourceIDMap(terraformAwsProviderRepoPath, outputPath, resourceFileNames) + resourceIDs, err := terraform.GenerateResourceIDMap(terraformAwsProviderRepoPath, outputPathResource, resourceFileNames) if err != nil { log.WithError(err).Fatal("failed to generate map of resource type -> resource ID") } - resourceTypesWithTags, err := terraform.GenerateResourceTypesWithTagsList(resourceTypes, outputPath) + resourceTypesWithTags, err := terraform.GenerateResourceTypesWithTagsList(resourceTypes, outputPathResource) if err != nil { log.WithError(err).Fatal("failed to generate list of resource type that support tags") } @@ -51,46 +89,43 @@ func main() { log.WithError(err).Fatal("failed to load AWS APIs") } - terraformServices := aws.ServicesCoveredByTerraform(resourceServices) + services := aws.ResourceTypesForAWSServices(resourceServices) servicePkgNames := aws.ServicePkgNames(apis) - log.Infof("AWS services covered by Terraform: %d/%d", - len(terraformServices), len(servicePkgNames)) + log.Infof("AWS generatedServices covered by Terraform: %d/%d", len(services), len(servicePkgNames)) - log.Debugf("AWS services not covered:") - diff := util.Difference(servicePkgNames, terraformServices) - for _, d := range diff { - log.Debugf("\t%s", d) - } + /* + log.Debugf("AWS generatedServices not covered:") + diff := util.Difference(servicePkgNames, services) + for _, d := range diff { + log.Debugf("\t%s", d) + } - log.Warn("AWS services used by Terraform that are named differently in AWS API v2:") - diff = util.Difference(terraformServices, servicePkgNames) - for _, d := range diff { - log.Warnf("\t: %s", d) - } - - err = aws.GenerateClient("../aws", servicePkgNames) - if err != nil { - log.WithError(err).Fatal("Failed to write AWS client") - } + log.Warn("AWS generatedServices used by Terraform that are named differently in AWS API v2:") + diff = util.Difference(services, servicePkgNames) + for _, d := range diff { + log.Warnf("\t: %s", d) + } + */ - listFunctionNames, genResourceInfos := aws.GenerateListFunctions("../aws", - resourceServices, resourceIDs, resourceTypesWithTags, apis) + aws.GenerateClient(outputPathAWS, servicePkgNames) - log.Infof("Generated list functions: %d", len(listFunctionNames)) + generatedServices := aws.GenerateListFunctions(outputPathAWS, services, resourceIDs, resourceTypesWithTags, apis) - err = terraform.GenerateListResourcesByTypeFunction("../aws", listFunctionNames) - if err != nil { - log.WithError(err).Fatal("failed to generate list resource function by type") + var rTypes []aws.ResourceType + for _, service := range generatedServices { + for _, rType := range service.TerraformResourceTypes { + rTypes = append(rTypes, rType) + } } - err = aws.GenerateSupportedResourceTypeList(outputPath, listFunctionNames) - if err != nil { - log.WithError(err).Fatal("failed to generate list supported resource types") - } + sort.Slice(rTypes, func(i, j int) bool { + return rTypes[i].Name < rTypes[j].Name + }) - err = aws.WriteReadme("..", genResourceInfos) - if err != nil { - log.WithError(err).Fatal("failed to generate README") - } + terraform.GenerateListResourcesByTypeFunction(outputPathAWS, rTypes) + aws.GenerateListOfSupportedResourceTypes(outputPathResource, rTypes) + aws.GenerateReadme("..", generatedServices, len(rTypes)) + + log.Infof("Generated list functions: %d", len(rTypes)) } diff --git a/gen/terraform/list_resources.go b/gen/terraform/list_resources.go index 9244cfe..d352f2c 100644 --- a/gen/terraform/list_resources.go +++ b/gen/terraform/list_resources.go @@ -4,34 +4,33 @@ package terraform import ( "bytes" - "fmt" "path/filepath" "strings" "text/template" + "github.com/jckuester/awsls/gen/aws" + "github.com/jckuester/awsls/gen/util" ) //GenerateListResourcesByTypeFunction generates a function to list all resources of a given Terraform resource type. -func GenerateListResourcesByTypeFunction(outputPath string, listFunctionNames map[string]string) error { +func GenerateListResourcesByTypeFunction(outputPath string, rTypes []aws.ResourceType) { err := util.WriteGoFile( filepath.Join(outputPath, "list.go"), util.CodeLayout, "", "aws", - listByTypeGoCode(listFunctionNames), + listByTypeGoCode(rTypes), ) if err != nil { - return fmt.Errorf("failed to write Go code to file: %s", err) + panic(err) } - - return nil } -func listByTypeGoCode(listFunctionNames map[string]string) string { +func listByTypeGoCode(rTypes []aws.ResourceType) string { var buf bytes.Buffer - err := listByTypeTmpl.Execute(&buf, listFunctionNames) + err := listByTypeTmpl.Execute(&buf, rTypes) if err != nil { panic(err) } @@ -59,8 +58,8 @@ type Resource struct { func ListResourcesByType(client *Client, resourceType string) ([]Resource, error) { switch resourceType { - {{ range $key, $value := . }}case "{{ $key }}": - return List{{ $value }}(client) + {{ range . }}case "{{ .Name }}": + return {{ .ListFunctionName }}(client) {{ end }}default: return nil, fmt.Errorf("resource type is not (yet) supported: %s", resourceType) } diff --git a/gen/terraform/resource_id.go b/gen/terraform/resource_id.go index 2e4c3cb..3d4e37c 100644 --- a/gen/terraform/resource_id.go +++ b/gen/terraform/resource_id.go @@ -8,7 +8,6 @@ import ( "go/ast" "go/parser" "go/token" - "os" "path/filepath" "strings" "text/template" @@ -30,15 +29,13 @@ func GenerateResourceIDMap(providerRepoPath, outputPath string, resourceFileName resourceIDs[rType] = resourceID } - err := writeResourceIDs(outputPath, resourceIDs) - if err != nil { - return nil, err - } + writeResourceIDs(outputPath, resourceIDs) log.WithField("length", len(resourceIDs)).Infof("Generated map of resource type -> resource ID") return resourceIDs, nil } + func GetResourceID(providerRepoPath, fileName string) (string, error) { fset := token.NewFileSet() @@ -183,24 +180,18 @@ func GetResourceID(providerRepoPath, fileName string) (string, error) { return "", fmt.Errorf("no ID found for resource type") } -func writeResourceIDs(outputPath string, resourceIDs map[string]string) error { - err := os.MkdirAll(outputPath, 0775) - if err != nil { - return fmt.Errorf("failed to create directory: %s", err) - } - - err = util.WriteGoFile( +func writeResourceIDs(outputPath string, resourceIDs map[string]string) { + err := util.WriteGoFile( filepath.Join(outputPath, "ids.go"), util.CodeLayout, "", "resource", resourceIDsGoCode(resourceIDs), ) + if err != nil { - return fmt.Errorf("failed to write Go code to file: %s", err) + panic(err) } - - return nil } func resourceIDsGoCode(resourceIDs map[string]string) string { diff --git a/gen/terraform/resource_service.go b/gen/terraform/resource_service.go index f22d0d6..a83f292 100644 --- a/gen/terraform/resource_service.go +++ b/gen/terraform/resource_service.go @@ -7,7 +7,6 @@ import ( "fmt" "go/parser" "go/token" - "os" "path/filepath" "strings" "text/template" @@ -18,17 +17,14 @@ import ( ) func GenerateResourceServiceMap(providerRepoPath string, outputPath string, resourceTypes []string, - resourceFileNames map[string]string) (map[string]string, error) { + resourceFileNames map[string]string) map[string]string { resourceServices := ResourceServices(providerRepoPath, resourceTypes, resourceFileNames) - err := writeResourceServices(outputPath, resourceServices) - if err != nil { - return nil, err - } + writeResourceServices(outputPath, resourceServices) log.WithField("length", len(resourceServices)).Infof("Generated map of resource type -> AWS service") - return resourceServices, nil + return resourceServices } // resourceService returns the AWS service that the Terraform resource belongs to. @@ -95,24 +91,18 @@ func resourceService(providerRepoPath, resourceType, resourceFileName string) (s return "", fmt.Errorf("no service candidate found") } -func writeResourceServices(outputPath string, resourceServices map[string]string) error { - err := os.MkdirAll(outputPath, 0775) - if err != nil { - return fmt.Errorf("failed to create directory: %s", err) - } - - err = util.WriteGoFile( +func writeResourceServices(outputPath string, resourceServices map[string]string) { + err := util.WriteGoFile( filepath.Join(outputPath, "services.go"), util.CodeLayout, "", "resource", resourceServicesGoCode(resourceServices), ) + if err != nil { - return fmt.Errorf("failed to write Go code to file: %s", err) + panic(err) } - - return nil } func resourceServicesGoCode(terraformTypes map[string]string) string { diff --git a/gen/terraform/resource_type.go b/gen/terraform/resource_type.go index 96692fd..bb791c0 100644 --- a/gen/terraform/resource_type.go +++ b/gen/terraform/resource_type.go @@ -8,7 +8,6 @@ import ( "go/ast" "go/parser" "go/token" - "os" "path/filepath" "sort" "strings" @@ -26,10 +25,7 @@ func GenerateResourceTypeList(providerRepoPath, outputPath string) ([]string, er return nil, err } - err = writeResourceTypes(outputPath, resourceTypes) - if err != nil { - return nil, err - } + writeResourceTypes(outputPath, resourceTypes) log.WithField("length", len(resourceTypes)).Infof("Generated list of Terraform AWS resource types") return resourceTypes, nil @@ -86,18 +82,10 @@ func ResourceTypes(providerRepoPath string) ([]string, error) { return result, nil } -func writeResourceTypes(outputPath string, resourceTypes []string) error { - err := os.MkdirAll(outputPath, 0775) - if err != nil { - return fmt.Errorf("failed to create directory: %s", err) - } +func writeResourceTypes(outputPath string, resourceTypes []string) { + code := resourceTypesGoCode(resourceTypes) - code, err := resourceTypesGoCode(resourceTypes) - if err != nil { - return fmt.Errorf("failed to generate Go code: %s", err) - } - - err = util.WriteGoFile( + err := util.WriteGoFile( filepath.Join(outputPath, "types.go"), util.CodeLayout, "", @@ -106,20 +94,18 @@ func writeResourceTypes(outputPath string, resourceTypes []string) error { ) if err != nil { - return fmt.Errorf("failed to write Go code to file: %s", err) + panic(err) } - - return nil } -func resourceTypesGoCode(terraformTypes []string) (string, error) { +func resourceTypesGoCode(terraformTypes []string) string { var buf bytes.Buffer err := resourceTypesTmpl.Execute(&buf, terraformTypes) if err != nil { - return "", err + panic(err) } - return strings.TrimSpace(buf.String()), nil + return strings.TrimSpace(buf.String()) } var resourceTypesTmpl = template.Must(template.New("resourceTypes").Parse(` diff --git a/gen/terraform/tags.go b/gen/terraform/tags.go index 7460a71..1b26cea 100644 --- a/gen/terraform/tags.go +++ b/gen/terraform/tags.go @@ -67,11 +67,6 @@ func GenerateResourceTypesWithTagsList(resourceTypes []string, outputPath string } func writeResourceTypesWithTags(outputPath string, resourceTypes []string) error { - err := os.MkdirAll(outputPath, 0775) - if err != nil { - return fmt.Errorf("failed to create directory: %s", err) - } - code, err := resourceTypesWithTagsGoCode(resourceTypes) if err != nil { return fmt.Errorf("failed to generate Go code: %s", err) diff --git a/gen/util/util.go b/gen/util/util.go index 31851f1..c697b79 100644 --- a/gen/util/util.go +++ b/gen/util/util.go @@ -59,3 +59,13 @@ func Difference(slice1 []string, slice2 []string) []string { return diff } + +func Contains(s []string, e string) bool { + for _, a := range s { + if a == e { + return true + } + } + + return false +} diff --git a/go.sum b/go.sum index e1531cf..20c7a8a 100644 --- a/go.sum +++ b/go.sum @@ -1060,6 +1060,7 @@ gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/resource/generate.go b/resource/generate.go index 1a9c8e9..c30c7a6 100644 --- a/resource/generate.go +++ b/resource/generate.go @@ -1,4 +1,5 @@ package resource //go:generate go run -tags codegen ../gen/main.go -//go:generate gofmt -s -w ../resource +//go:generate gofmt -s -w ../resource ../aws +//go:generate goimports -w ../resource ../aws diff --git a/resource/supported.go b/resource/supported.go index 99e6e4f..5cb56f5 100644 --- a/resource/supported.go +++ b/resource/supported.go @@ -29,6 +29,7 @@ var SupportedTypes = []string{ "aws_cloudformation_stack", "aws_cloudformation_stack_set", "aws_cloudhsm_v2_cluster", + "aws_cloudtrail", "aws_cloudwatch_dashboard", "aws_cloudwatch_event_bus", "aws_cloudwatch_log_destination", @@ -236,6 +237,8 @@ var SupportedTypes = []string{ "aws_wafregional_xss_match_set", "aws_wafv2_web_acl_logging_configuration", "aws_worklink_fleet", + "aws_workspaces_directory", "aws_workspaces_ip_group", + "aws_workspaces_workspace", "aws_xray_group", }