From 903a0497acbeaadc2452d1e70301c5496c6ded7d Mon Sep 17 00:00:00 2001 From: talset Date: Wed, 13 Nov 2019 11:56:53 +0100 Subject: [PATCH 1/4] aws: Add LaunchConfiguration LaunchTemplate AutoscalingGroup LaunchConfiguration is now supported on cycloid-raws. LaunchTemplate is now supported on cycloid-raws. AutoscalingGroup is now supported on cycloid-raws. import.go: All read error is now ignored (continue) and logged. --- aws/resources.go | 69 ++++++++++++++++++++++++++++++++++++++++++++++ cmd/aws.go | 1 + provider/import.go | 13 ++++----- 3 files changed, 75 insertions(+), 8 deletions(-) diff --git a/aws/resources.go b/aws/resources.go index 6f51e58c4e..0678da2812 100644 --- a/aws/resources.go +++ b/aws/resources.go @@ -88,6 +88,9 @@ const ( // SESEventDestination SESIdentityNotificationTopic SESTemplate + LaunchConfiguration + LaunchTemplate + AutoscalingGroup ) type rtFn func(ctx context.Context, a *aws, resourceType string, tags []tag.Tag) ([]provider.Resource, error) @@ -148,6 +151,9 @@ var ( SESConfigurationSet: sesConfigurationSets, SESIdentityNotificationTopic: sesIdentityNotificationTopics, SESTemplate: sesTemplates, + LaunchConfiguration: launchConfigurations, + LaunchTemplate: launchtemplates, + AutoscalingGroup: autoscalinggroups, } ) @@ -1225,6 +1231,69 @@ func sesTemplates(ctx context.Context, a *aws, resourceType string, tags []tag.T return resources, nil } +func launchConfigurations(ctx context.Context, a *aws, resourceType string, tags []tag.Tag) ([]provider.Resource, error) { + launchConfigurations, err := a.awsr.GetLaunchConfigurations(ctx, nil) + + if err != nil { + return nil, err + } + + resources := make([]provider.Resource, 0) + for _, i := range launchConfigurations[a.Region()].LaunchConfigurations { + + r, err := initializeResource(a, *i.LaunchConfigurationName, resourceType) + if err != nil { + return nil, err + } + + resources = append(resources, r) + } + + return resources, nil +} + +func launchtemplates(ctx context.Context, a *aws, resourceType string, tags []tag.Tag) ([]provider.Resource, error) { + launchtemplates, err := a.awsr.GetLaunchTemplates(ctx, nil) + + if err != nil { + return nil, err + } + + resources := make([]provider.Resource, 0) + for _, i := range launchtemplates[a.Region()].LaunchTemplates { + + r, err := initializeResource(a, *i.LaunchTemplateId, resourceType) + if err != nil { + return nil, err + } + + resources = append(resources, r) + } + + return resources, nil +} + +func autoscalinggroups(ctx context.Context, a *aws, resourceType string, tags []tag.Tag) ([]provider.Resource, error) { + autoscalinggroups, err := a.awsr.GetAutoScalingGroups(ctx, nil) + + if err != nil { + return nil, err + } + + resources := make([]provider.Resource, 0) + for _, i := range autoscalinggroups[a.Region()].AutoScalingGroups { + + r, err := initializeResource(a, *i.AutoScalingGroupName, resourceType) + if err != nil { + return nil, err + } + + resources = append(resources, r) + } + + return resources, nil +} + func toEC2Filters(tags []tag.Tag) []*ec2.Filter { if len(tags) == 0 { return nil diff --git a/cmd/aws.go b/cmd/aws.go index ef5cd04c99..8f146740c7 100644 --- a/cmd/aws.go +++ b/cmd/aws.go @@ -100,6 +100,7 @@ func init() { awsCmd.Flags().String("secret-key", "", "Secret Key (required)") awsCmd.Flags().String("region", "", "Region to search in, for now * it's not supported (required)") + // Filter flags awsCmd.Flags().StringSliceVarP(&tags, "tags", "t", []string{}, "List of tags to filter with format 'NAME:VALUE'") } diff --git a/provider/import.go b/provider/import.go index 53af7b20d3..15cd5661ac 100644 --- a/provider/import.go +++ b/provider/import.go @@ -80,16 +80,13 @@ func Import(ctx context.Context, p Provider, hcl, tfstate writer.Writer, f *filt err = util.RetryDefault(func() error { return r.Read(f) }) if err != nil { cause := errors.Cause(err) + // All those ignored errors are types of errors to identify that something happend, but not that something went wrong // some of them are to skip the resource and others just informative - if cause != errcode.ErrProviderResourceNotRead && cause != errcode.ErrProviderResourceDoNotMatchTag && cause != errcode.ErrProviderResourceAutogenerated { - return errors.Wrapf(err, "could not read resource %s: ", r.Type()) - } - if cause == errcode.ErrProviderResourceNotRead || cause == errcode.ErrProviderResourceAutogenerated { - // As the resource could not be Read, meaning an ID == "" - // we'll continue to the next resource - continue - } + // we'll continue to the next resource + fmt.Fprintf(out, "\ncould not read resource: %s.%s\n", re.Type(), re.ID()) + logger.Log("error", cause) + continue } if hcl != nil { From cf922954d232f8e35283ee1cc2202b519502cb7b Mon Sep 17 00:00:00 2001 From: talset Date: Fri, 15 Nov 2019 16:45:55 +0100 Subject: [PATCH 2/4] aws/resourcetype_enumer.go: generate Generate resourcetype_enumer regarding new types * LaunchConfiguration * LaunchTemplate * AutoscalingGroup --- aws/resourcetype_enumer.go | 17 +++++++++++++---- provider/import.go | 8 ++++---- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/aws/resourcetype_enumer.go b/aws/resourcetype_enumer.go index 9979ec153f..5e375cacf2 100644 --- a/aws/resourcetype_enumer.go +++ b/aws/resourcetype_enumer.go @@ -6,11 +6,11 @@ import ( "fmt" ) -const _ResourceTypeName = "aws_instanceaws_vpcaws_security_groupaws_subnetaws_ebs_volumeaws_elasticache_clusteraws_elbaws_albaws_db_instanceaws_s3_bucketaws_cloudfront_distributionaws_cloudfront_origin_access_identityaws_cloudfront_public_keyaws_iam_account_aliasaws_iam_account_password_policyaws_iam_groupaws_iam_group_membershipaws_iam_group_policyaws_iam_group_policy_attachmentaws_iam_instance_profileaws_iam_openid_connect_provideraws_iam_policyaws_iam_roleaws_iam_role_policyaws_iam_role_policy_attachmentaws_iam_saml_provideraws_iam_server_certificateaws_iam_useraws_iam_user_group_membershipaws_iam_user_policyaws_iam_user_policy_attachmentaws_route53_delegation_setaws_route53_health_checkaws_route53_query_logaws_route53_recordaws_route53_zoneaws_route53_zone_associationaws_route53_resolver_endpointaws_route53_resolver_rule_associationaws_ses_active_receipt_rule_setaws_ses_domain_identityaws_ses_domain_identity_verificationaws_ses_domain_dkimaws_ses_domain_mail_fromaws_ses_receipt_filteraws_ses_receipt_ruleaws_ses_receipt_rule_setaws_ses_configuration_setaws_ses_identity_notification_topicaws_ses_template" +const _ResourceTypeName = "aws_instanceaws_vpcaws_security_groupaws_subnetaws_ebs_volumeaws_elasticache_clusteraws_elbaws_albaws_db_instanceaws_s3_bucketaws_cloudfront_distributionaws_cloudfront_origin_access_identityaws_cloudfront_public_keyaws_iam_account_aliasaws_iam_account_password_policyaws_iam_groupaws_iam_group_membershipaws_iam_group_policyaws_iam_group_policy_attachmentaws_iam_instance_profileaws_iam_openid_connect_provideraws_iam_policyaws_iam_roleaws_iam_role_policyaws_iam_role_policy_attachmentaws_iam_saml_provideraws_iam_server_certificateaws_iam_useraws_iam_user_group_membershipaws_iam_user_policyaws_iam_user_policy_attachmentaws_route53_delegation_setaws_route53_health_checkaws_route53_query_logaws_route53_recordaws_route53_zoneaws_route53_zone_associationaws_route53_resolver_endpointaws_route53_resolver_rule_associationaws_ses_active_receipt_rule_setaws_ses_domain_identityaws_ses_domain_identity_verificationaws_ses_domain_dkimaws_ses_domain_mail_fromaws_ses_receipt_filteraws_ses_receipt_ruleaws_ses_receipt_rule_setaws_ses_configuration_setaws_ses_identity_notification_topicaws_ses_templateaws_launch_configurationaws_launch_templateaws_autoscaling_group" -var _ResourceTypeIndex = [...]uint16{0, 12, 19, 37, 47, 61, 84, 91, 98, 113, 126, 153, 190, 215, 236, 267, 280, 304, 324, 355, 379, 410, 424, 436, 455, 485, 506, 532, 544, 573, 592, 622, 648, 672, 693, 711, 727, 755, 784, 821, 852, 875, 911, 930, 954, 976, 996, 1020, 1045, 1080, 1096} +var _ResourceTypeIndex = [...]uint16{0, 12, 19, 37, 47, 61, 84, 91, 98, 113, 126, 153, 190, 215, 236, 267, 280, 304, 324, 355, 379, 410, 424, 436, 455, 485, 506, 532, 544, 573, 592, 622, 648, 672, 693, 711, 727, 755, 784, 821, 852, 875, 911, 930, 954, 976, 996, 1020, 1045, 1080, 1096, 1120, 1139, 1160} -const _ResourceTypeLowerName = "aws_instanceaws_vpcaws_security_groupaws_subnetaws_ebs_volumeaws_elasticache_clusteraws_elbaws_albaws_db_instanceaws_s3_bucketaws_cloudfront_distributionaws_cloudfront_origin_access_identityaws_cloudfront_public_keyaws_iam_account_aliasaws_iam_account_password_policyaws_iam_groupaws_iam_group_membershipaws_iam_group_policyaws_iam_group_policy_attachmentaws_iam_instance_profileaws_iam_openid_connect_provideraws_iam_policyaws_iam_roleaws_iam_role_policyaws_iam_role_policy_attachmentaws_iam_saml_provideraws_iam_server_certificateaws_iam_useraws_iam_user_group_membershipaws_iam_user_policyaws_iam_user_policy_attachmentaws_route53_delegation_setaws_route53_health_checkaws_route53_query_logaws_route53_recordaws_route53_zoneaws_route53_zone_associationaws_route53_resolver_endpointaws_route53_resolver_rule_associationaws_ses_active_receipt_rule_setaws_ses_domain_identityaws_ses_domain_identity_verificationaws_ses_domain_dkimaws_ses_domain_mail_fromaws_ses_receipt_filteraws_ses_receipt_ruleaws_ses_receipt_rule_setaws_ses_configuration_setaws_ses_identity_notification_topicaws_ses_template" +const _ResourceTypeLowerName = "aws_instanceaws_vpcaws_security_groupaws_subnetaws_ebs_volumeaws_elasticache_clusteraws_elbaws_albaws_db_instanceaws_s3_bucketaws_cloudfront_distributionaws_cloudfront_origin_access_identityaws_cloudfront_public_keyaws_iam_account_aliasaws_iam_account_password_policyaws_iam_groupaws_iam_group_membershipaws_iam_group_policyaws_iam_group_policy_attachmentaws_iam_instance_profileaws_iam_openid_connect_provideraws_iam_policyaws_iam_roleaws_iam_role_policyaws_iam_role_policy_attachmentaws_iam_saml_provideraws_iam_server_certificateaws_iam_useraws_iam_user_group_membershipaws_iam_user_policyaws_iam_user_policy_attachmentaws_route53_delegation_setaws_route53_health_checkaws_route53_query_logaws_route53_recordaws_route53_zoneaws_route53_zone_associationaws_route53_resolver_endpointaws_route53_resolver_rule_associationaws_ses_active_receipt_rule_setaws_ses_domain_identityaws_ses_domain_identity_verificationaws_ses_domain_dkimaws_ses_domain_mail_fromaws_ses_receipt_filteraws_ses_receipt_ruleaws_ses_receipt_rule_setaws_ses_configuration_setaws_ses_identity_notification_topicaws_ses_templateaws_launch_configurationaws_launch_templateaws_autoscaling_group" func (i ResourceType) String() string { i -= 1 @@ -20,7 +20,7 @@ func (i ResourceType) String() string { return _ResourceTypeName[_ResourceTypeIndex[i]:_ResourceTypeIndex[i+1]] } -var _ResourceTypeValues = []ResourceType{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50} +var _ResourceTypeValues = []ResourceType{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53} var _ResourceTypeNameToValueMap = map[string]ResourceType{ _ResourceTypeName[0:12]: 1, @@ -123,6 +123,12 @@ var _ResourceTypeNameToValueMap = map[string]ResourceType{ _ResourceTypeLowerName[1045:1080]: 49, _ResourceTypeName[1080:1096]: 50, _ResourceTypeLowerName[1080:1096]: 50, + _ResourceTypeName[1096:1120]: 51, + _ResourceTypeLowerName[1096:1120]: 51, + _ResourceTypeName[1120:1139]: 52, + _ResourceTypeLowerName[1120:1139]: 52, + _ResourceTypeName[1139:1160]: 53, + _ResourceTypeLowerName[1139:1160]: 53, } var _ResourceTypeNames = []string{ @@ -176,6 +182,9 @@ var _ResourceTypeNames = []string{ _ResourceTypeName[1020:1045], _ResourceTypeName[1045:1080], _ResourceTypeName[1080:1096], + _ResourceTypeName[1096:1120], + _ResourceTypeName[1120:1139], + _ResourceTypeName[1139:1160], } // ResourceTypeString retrieves an enum value from the enum constants string name. diff --git a/provider/import.go b/provider/import.go index 15cd5661ac..42c4b53181 100644 --- a/provider/import.go +++ b/provider/import.go @@ -81,11 +81,11 @@ func Import(ctx context.Context, p Provider, hcl, tfstate writer.Writer, f *filt if err != nil { cause := errors.Cause(err) - // All those ignored errors are types of errors to identify that something happend, but not that something went wrong - // some of them are to skip the resource and others just informative - // we'll continue to the next resource - fmt.Fprintf(out, "\ncould not read resource: %s.%s\n", re.Type(), re.ID()) + // Errors are ignored. If a resource is invalid we assume it can be skipped, it can be related to inconsistencies in deployed resources. + // So instead of failing and stopping execution we ignore them and continue (we log them if -v is specified) + logger.Log("error", cause) + continue } From 1ae0ab60a84de9e2947caea9aada1a00c4081736 Mon Sep 17 00:00:00 2001 From: talset Date: Wed, 27 Nov 2019 18:17:24 +0100 Subject: [PATCH 3/4] CHANGELOG.md: Update Added and Changed --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26280949dc..40f8d40d7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,11 +2,18 @@ ### Added +- aws resource: `aws_launch_configuration`, `aws_launch_template` and `aws_autoscaling_group` + ([PR #68](https://github.com/cycloidio/terracognita/pull/68)) - google resource: compute instance ([PR #56](https://github.com/cycloidio/terracognita/pull/56)) - google resource: compute networks and compute firewalls ([PR #61](https://github.com/cycloidio/terracognita/pull/61)) +### Changed + +- During import if a resource is invalid we assume it can be skipped + ([PR #68](https://github.com/cycloidio/terracognita/pull/68)) + ## Fixed - '--region' flag working for different subcommands From db6d0f9fc0944e3a391065ebfaef3b37f84c4893 Mon Sep 17 00:00:00 2001 From: talset Date: Thu, 28 Nov 2019 15:18:12 +0100 Subject: [PATCH 4/4] provider/import_test: fix expected func not called Since we continue on importing error, HCL and State are skipped for filtered/error resources. --- cmd/aws.go | 1 - go.mod | 2 +- provider/import_test.go | 4 ---- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/cmd/aws.go b/cmd/aws.go index 8f146740c7..ef5cd04c99 100644 --- a/cmd/aws.go +++ b/cmd/aws.go @@ -100,7 +100,6 @@ func init() { awsCmd.Flags().String("secret-key", "", "Secret Key (required)") awsCmd.Flags().String("region", "", "Region to search in, for now * it's not supported (required)") - // Filter flags awsCmd.Flags().StringSliceVarP(&tags, "tags", "t", []string{}, "List of tags to filter with format 'NAME:VALUE'") } diff --git a/go.mod b/go.mod index 32acea3b8f..d922e60d5b 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ require ( cloud.google.com/go/storage v1.0.0 // indirect github.com/aws/aws-sdk-go v1.25.4 github.com/chr4/pwgen v1.1.0 - github.com/cycloidio/raws v1.0.1 + github.com/cycloidio/raws v1.0.2 github.com/go-kit/kit v0.9.0 github.com/golang/mock v1.3.1 github.com/golang/snappy v0.0.1 // indirect diff --git a/provider/import_test.go b/provider/import_test.go index 1a960cd100..49f1573f2f 100644 --- a/provider/import_test.go +++ b/provider/import_test.go @@ -188,10 +188,8 @@ func TestImport(t *testing.T) { iamUser1.EXPECT().Read(f).Return(errcode.ErrProviderResourceDoNotMatchTag) iamUser2.EXPECT().Read(f).Return(nil) - iamUser1.EXPECT().HCL(hw).Return(nil) iamUser2.EXPECT().HCL(hw).Return(nil) - iamUser1.EXPECT().State(sw).Return(nil) iamUser2.EXPECT().State(sw).Return(nil) hw.EXPECT().Sync().Return(nil) @@ -231,7 +229,6 @@ func TestImport(t *testing.T) { iamUser1.EXPECT().Read(f).Return(errcode.ErrProviderResourceDoNotMatchTag) iamUser2.EXPECT().Read(f).Return(nil) - iamUser1.EXPECT().State(sw).Return(nil) iamUser2.EXPECT().State(sw).Return(nil) sw.EXPECT().Sync().Return(nil) @@ -270,7 +267,6 @@ func TestImport(t *testing.T) { iamUser1.EXPECT().Read(f).Return(errcode.ErrProviderResourceDoNotMatchTag) iamUser2.EXPECT().Read(f).Return(nil) - iamUser1.EXPECT().HCL(hw).Return(nil) iamUser2.EXPECT().HCL(hw).Return(nil) hw.EXPECT().Sync().Return(nil)