From 73d88e3fc2e925e089e6b0d14de74c06645ec735 Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Wed, 27 Jul 2022 11:28:02 -0400 Subject: [PATCH 1/9] Adding LogGroups --- client/mocks/mock_cloudwatchlogs.go | 40 +++++ client/services.go | 2 + docs/tables/aws_cloudwatchlogs_log_groups.md | 15 ++ resources/provider/provider.go | 1 + .../services/cloudwatchlogs/log_groups.go | 139 ++++++++++++++++++ .../cloudwatchlogs/log_groups_mock_test.go | 32 ++++ 6 files changed, 229 insertions(+) create mode 100644 docs/tables/aws_cloudwatchlogs_log_groups.md create mode 100644 resources/services/cloudwatchlogs/log_groups.go create mode 100644 resources/services/cloudwatchlogs/log_groups_mock_test.go diff --git a/client/mocks/mock_cloudwatchlogs.go b/client/mocks/mock_cloudwatchlogs.go index 720c9f4be..da6515fc3 100644 --- a/client/mocks/mock_cloudwatchlogs.go +++ b/client/mocks/mock_cloudwatchlogs.go @@ -35,6 +35,46 @@ func (m *MockCloudwatchLogsClient) EXPECT() *MockCloudwatchLogsClientMockRecorde return m.recorder } +// DescribeLogGroups mocks base method. +func (m *MockCloudwatchLogsClient) DescribeLogGroups(arg0 context.Context, arg1 *cloudwatchlogs.DescribeLogGroupsInput, arg2 ...func(*cloudwatchlogs.Options)) (*cloudwatchlogs.DescribeLogGroupsOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "DescribeLogGroups", varargs...) + ret0, _ := ret[0].(*cloudwatchlogs.DescribeLogGroupsOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DescribeLogGroups indicates an expected call of DescribeLogGroups. +func (mr *MockCloudwatchLogsClientMockRecorder) DescribeLogGroups(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeLogGroups", reflect.TypeOf((*MockCloudwatchLogsClient)(nil).DescribeLogGroups), varargs...) +} + +// DescribeLogStreams mocks base method. +func (m *MockCloudwatchLogsClient) DescribeLogStreams(arg0 context.Context, arg1 *cloudwatchlogs.DescribeLogStreamsInput, arg2 ...func(*cloudwatchlogs.Options)) (*cloudwatchlogs.DescribeLogStreamsOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "DescribeLogStreams", varargs...) + ret0, _ := ret[0].(*cloudwatchlogs.DescribeLogStreamsOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DescribeLogStreams indicates an expected call of DescribeLogStreams. +func (mr *MockCloudwatchLogsClientMockRecorder) DescribeLogStreams(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeLogStreams", reflect.TypeOf((*MockCloudwatchLogsClient)(nil).DescribeLogStreams), varargs...) +} + // DescribeMetricFilters mocks base method. func (m *MockCloudwatchLogsClient) DescribeMetricFilters(arg0 context.Context, arg1 *cloudwatchlogs.DescribeMetricFiltersInput, arg2 ...func(*cloudwatchlogs.Options)) (*cloudwatchlogs.DescribeMetricFiltersOutput, error) { m.ctrl.T.Helper() diff --git a/client/services.go b/client/services.go index d67ff8dd0..681a5b4c2 100644 --- a/client/services.go +++ b/client/services.go @@ -172,6 +172,8 @@ type CloudwatchClient interface { //go:generate mockgen -package=mocks -destination=./mocks/mock_cloudwatchlogs.go . CloudwatchLogsClient type CloudwatchLogsClient interface { DescribeMetricFilters(ctx context.Context, params *cloudwatchlogs.DescribeMetricFiltersInput, optFns ...func(*cloudwatchlogs.Options)) (*cloudwatchlogs.DescribeMetricFiltersOutput, error) + DescribeLogGroups(ctx context.Context, params *cloudwatchlogs.DescribeLogGroupsInput, optFns ...func(*cloudwatchlogs.Options)) (*cloudwatchlogs.DescribeLogGroupsOutput, error) + DescribeLogStreams(ctx context.Context, params *cloudwatchlogs.DescribeLogStreamsInput, optFns ...func(*cloudwatchlogs.Options)) (*cloudwatchlogs.DescribeLogStreamsOutput, error) } //go:generate mockgen -package=mocks -destination=./mocks/mock_cloudformation.go . CloudFormationClient diff --git a/docs/tables/aws_cloudwatchlogs_log_groups.md b/docs/tables/aws_cloudwatchlogs_log_groups.md new file mode 100644 index 000000000..6a5602e67 --- /dev/null +++ b/docs/tables/aws_cloudwatchlogs_log_groups.md @@ -0,0 +1,15 @@ + +# Table: aws_cloudwatchlogs_log_groups +CloudWatch Logs enables you to centralize the logs from all of your systems, applications, and AWS services that you use, in a single, highly scalable service. Log groups define groups of log streams that share the same retention, monitoring, and access control settings. Each log stream has to belong to one log group. +## Columns +| Name | Type | Description | +| ------------- | ------------- | ----- | +|account_id|text|The AWS Account ID of the resource.| +|region|text|The AWS Region of the resource.| +|arn|text|The ARN (Amazon Resource Name) for the distribution| +|creation_time|bigint|The creation time of the log group, expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC.| +|kms_key_id|text|The Amazon Resource Name (ARN) of the CMK to use when encrypting log data.| +|name|text|The name of the log group.| +|metric_filter_count|bigint|The number of metric filters| +|retention_in_days|bigint|The number of days to retain the log events in the specified log group. Possible values are: 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, 2192, 2557, 2922, 3288, and 3653.| +|stored_bytes|bigint|The number of bytes stored| diff --git a/resources/provider/provider.go b/resources/provider/provider.go index 68a10786c..fd9d19dcb 100644 --- a/resources/provider/provider.go +++ b/resources/provider/provider.go @@ -109,6 +109,7 @@ func Provider() *provider.Provider { "cloudtrail.trails": cloudtrail.CloudtrailTrails(), "cloudwatch.alarms": cloudwatch.CloudwatchAlarms(), "cloudwatchlogs.filters": cloudwatchlogs.CloudwatchlogsFilters(), + "cloudwatchlogs.log_groups": cloudwatchlogs.CloudwatchlogsLogGroups(), "codebuild.projects": codebuild.CodebuildProjects(), "codepipeline.pipelines": codepipeline.Pipelines(), "codepipeline.webhooks": codepipeline.Webhooks(), diff --git a/resources/services/cloudwatchlogs/log_groups.go b/resources/services/cloudwatchlogs/log_groups.go new file mode 100644 index 000000000..03a83ccfa --- /dev/null +++ b/resources/services/cloudwatchlogs/log_groups.go @@ -0,0 +1,139 @@ +package cloudwatchlogs + +import ( + "context" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs" + "github.com/cloudquery/cq-provider-aws/client" + "github.com/cloudquery/cq-provider-sdk/provider/diag" + "github.com/cloudquery/cq-provider-sdk/provider/schema" +) + +func CloudwatchlogsLogGroups() *schema.Table { + return &schema.Table{ + Name: "aws_cloudwatchlogs_log_groups", + Description: "CloudWatch Logs enables you to centralize the logs from all of your systems, applications, and AWS services that you use, in a single, highly scalable service. Log groups define groups of log streams that share the same retention, monitoring, and access control settings. Each log stream has to belong to one log group. ", + Resolver: fetchCloudwatchlogsLogGroups, + Multiplex: client.ServiceAccountRegionMultiplexer("logs"), + IgnoreError: client.IgnoreAccessDeniedServiceDisabled, + DeleteFilter: client.DeleteAccountRegionFilter, + Options: schema.TableCreationOptions{PrimaryKeys: []string{"account_id", "region", "name"}}, + IgnoreInTests: true, + Columns: []schema.Column{ + { + Name: "account_id", + Description: "The AWS Account ID of the resource.", + Type: schema.TypeString, + Resolver: client.ResolveAWSAccount, + }, + { + Name: "region", + Description: "The AWS Region of the resource.", + Type: schema.TypeString, + Resolver: client.ResolveAWSRegion, + }, + { + Name: "arn", + Description: "The ARN (Amazon Resource Name) for the distribution", + Type: schema.TypeString, + }, + { + Name: "creation_time", + Description: "The creation time of the log group, expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC.", + Type: schema.TypeBigInt, + }, + { + Name: "kms_key_id", + Description: "The Amazon Resource Name (ARN) of the CMK to use when encrypting log data.", + Type: schema.TypeString, + Resolver: schema.PathResolver("KmsKeyId"), + }, + { + Name: "name", + Description: "The name of the log group.", + Type: schema.TypeString, + Resolver: schema.PathResolver("LogGroupName"), + }, + { + Name: "metric_filter_count", + Description: "The number of metric filters", + Type: schema.TypeBigInt, + }, + { + Name: "retention_in_days", + Description: "The number of days to retain the log events in the specified log group. Possible values are: 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, 2192, 2557, 2922, 3288, and 3653.", + Type: schema.TypeBigInt, + }, + { + Name: "stored_bytes", + Description: "The number of bytes stored", + Type: schema.TypeBigInt, + }, + }, + // Relations: []*schema.Table{ + // { + // Name: "aws_cloudwatchlogs_log_group_transformations", + // Description: "Indicates how to transform ingested log events to metric data in a CloudWatch metric.", + // Resolver: fetchCloudwatchlogsLogGroupsTransformations, + // IgnoreInTests: true, + // Columns: []schema.Column{ + // { + // Name: "log_group_cq_id", + // Description: "Unique CloudQuery ID of aws_cloudwatchlogs_log_groups table (FK)", + // Type: schema.TypeUUID, + // Resolver: schema.ParentIdResolver, + // }, + // { + // Name: "metric_name", + // Description: "The name of the CloudWatch metric.", + // Type: schema.TypeString, + // }, + // { + // Name: "metric_namespace", + // Description: "A custom namespace to contain your metric in CloudWatch.", + // Type: schema.TypeString, + // }, + // { + // Name: "metric_value", + // Description: "The value to publish to the CloudWatch metric when a filter pattern matches a log event.", + // Type: schema.TypeString, + // }, + // { + // Name: "default_value", + // Description: "(Optional) The value to emit when a filter pattern does not match a log event.", + // Type: schema.TypeFloat, + // }, + // }, + // }, + // }, + } +} + +// ==================================================================================================================== +// Table Resolver Functions +// ==================================================================================================================== +func fetchCloudwatchlogsLogGroups(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- interface{}) error { + var config cloudwatchlogs.DescribeLogGroupsInput + c := meta.(*client.Client) + svc := c.Services().CloudwatchLogs + for { + response, err := svc.DescribeLogGroups(ctx, &config, func(options *cloudwatchlogs.Options) { + options.Region = c.Region + }) + if err != nil { + return diag.WrapError(err) + } + res <- response.LogGroups + if aws.ToString(response.NextToken) == "" { + break + } + config.NextToken = response.NextToken + } + return nil +} + +// func fetchCloudwatchlogsLogGroupsTransformations(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- interface{}) error { +// res <- parent.Item.(types.LogGroup) +// return nil +// } diff --git a/resources/services/cloudwatchlogs/log_groups_mock_test.go b/resources/services/cloudwatchlogs/log_groups_mock_test.go new file mode 100644 index 000000000..6f3129236 --- /dev/null +++ b/resources/services/cloudwatchlogs/log_groups_mock_test.go @@ -0,0 +1,32 @@ +package cloudwatchlogs + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs" + "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs/types" + "github.com/bxcodec/faker" + "github.com/cloudquery/cq-provider-aws/client" + "github.com/cloudquery/cq-provider-aws/client/mocks" + "github.com/golang/mock/gomock" +) + +func buildCloudwatchLogsLogGroupsMock(t *testing.T, ctrl *gomock.Controller) client.Services { + m := mocks.NewMockCloudwatchLogsClient(ctrl) + l := types.LogGroup{} + err := faker.FakeData(&l) + if err != nil { + t.Fatal(err) + } + m.EXPECT().DescribeLogGroups(gomock.Any(), gomock.Any(), gomock.Any()).Return( + &cloudwatchlogs.DescribeLogGroupsOutput{ + LogGroups: []types.LogGroup{l}, + }, nil) + return client.Services{ + CloudwatchLogs: m, + } +} + +func TestCloudwatchlogsLogGroups(t *testing.T) { + client.AwsMockTestHelper(t, CloudwatchlogsLogGroups(), buildCloudwatchLogsLogGroupsMock, client.TestOptions{}) +} From 80eea398047655442d3d9e24d1b566335e4dafd4 Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Wed, 27 Jul 2022 14:03:48 -0400 Subject: [PATCH 2/9] Log Streams related resource --- .../aws_cloudwatchlogs_log_group_streams.md | 14 +++ docs/tables/aws_cloudwatchlogs_log_groups.md | 2 +- .../services/cloudwatchlogs/log_groups.go | 118 ++++++++++++------ 3 files changed, 92 insertions(+), 42 deletions(-) create mode 100644 docs/tables/aws_cloudwatchlogs_log_group_streams.md diff --git a/docs/tables/aws_cloudwatchlogs_log_group_streams.md b/docs/tables/aws_cloudwatchlogs_log_group_streams.md new file mode 100644 index 000000000..710be4dcd --- /dev/null +++ b/docs/tables/aws_cloudwatchlogs_log_group_streams.md @@ -0,0 +1,14 @@ + +# Table: aws_cloudwatchlogs_log_group_streams +The Log Streams connected to a particular Log Group, each of which is a sequence of log events from a single emitter of logs +## Columns +| Name | Type | Description | +| ------------- | ------------- | ----- | +|log_group_cq_id|uuid|Unique CloudQuery ID of aws_cloudwatchlogs_log_groups table (FK)| +|arn|text|The ARN (Amazon Resource Name) for the stream.| +|creation_time|bigint|The creation time of the log group, expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC.| +|first_event_timestamp|bigint|The time of the first event, expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC.| +|last_event_timestamp|bigint|The time of the most recent log event in the log stream in CloudWatch Logs. This number is expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC. | +|last_ingestion_time|bigint|The ingestion time, expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC.| +|log_stream_name|text|The name of the CloudWatch metric.| +|stored_bytes|bigint|The number of bytes stored.| diff --git a/docs/tables/aws_cloudwatchlogs_log_groups.md b/docs/tables/aws_cloudwatchlogs_log_groups.md index 6a5602e67..4f3178c62 100644 --- a/docs/tables/aws_cloudwatchlogs_log_groups.md +++ b/docs/tables/aws_cloudwatchlogs_log_groups.md @@ -6,7 +6,7 @@ CloudWatch Logs enables you to centralize the logs from all of your systems, app | ------------- | ------------- | ----- | |account_id|text|The AWS Account ID of the resource.| |region|text|The AWS Region of the resource.| -|arn|text|The ARN (Amazon Resource Name) for the distribution| +|arn|text|The ARN (Amazon Resource Name) for the group.| |creation_time|bigint|The creation time of the log group, expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC.| |kms_key_id|text|The Amazon Resource Name (ARN) of the CMK to use when encrypting log data.| |name|text|The name of the log group.| diff --git a/resources/services/cloudwatchlogs/log_groups.go b/resources/services/cloudwatchlogs/log_groups.go index 03a83ccfa..85be240de 100644 --- a/resources/services/cloudwatchlogs/log_groups.go +++ b/resources/services/cloudwatchlogs/log_groups.go @@ -5,6 +5,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs" + "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs/types" "github.com/cloudquery/cq-provider-aws/client" "github.com/cloudquery/cq-provider-sdk/provider/diag" "github.com/cloudquery/cq-provider-sdk/provider/schema" @@ -35,7 +36,7 @@ func CloudwatchlogsLogGroups() *schema.Table { }, { Name: "arn", - Description: "The ARN (Amazon Resource Name) for the distribution", + Description: "The ARN (Amazon Resource Name) for the group.", Type: schema.TypeString, }, { @@ -71,42 +72,57 @@ func CloudwatchlogsLogGroups() *schema.Table { Type: schema.TypeBigInt, }, }, - // Relations: []*schema.Table{ - // { - // Name: "aws_cloudwatchlogs_log_group_transformations", - // Description: "Indicates how to transform ingested log events to metric data in a CloudWatch metric.", - // Resolver: fetchCloudwatchlogsLogGroupsTransformations, - // IgnoreInTests: true, - // Columns: []schema.Column{ - // { - // Name: "log_group_cq_id", - // Description: "Unique CloudQuery ID of aws_cloudwatchlogs_log_groups table (FK)", - // Type: schema.TypeUUID, - // Resolver: schema.ParentIdResolver, - // }, - // { - // Name: "metric_name", - // Description: "The name of the CloudWatch metric.", - // Type: schema.TypeString, - // }, - // { - // Name: "metric_namespace", - // Description: "A custom namespace to contain your metric in CloudWatch.", - // Type: schema.TypeString, - // }, - // { - // Name: "metric_value", - // Description: "The value to publish to the CloudWatch metric when a filter pattern matches a log event.", - // Type: schema.TypeString, - // }, - // { - // Name: "default_value", - // Description: "(Optional) The value to emit when a filter pattern does not match a log event.", - // Type: schema.TypeFloat, - // }, - // }, - // }, - // }, + Relations: []*schema.Table{ + { + Name: "aws_cloudwatchlogs_log_group_streams", + Description: "The Log Streams connected to a particular Log Group, each of which is a sequence of log events from a single emitter of logs", + Resolver: fetchCloudwatchlogsLogStreams, + IgnoreInTests: true, + Columns: []schema.Column{ + { + Name: "log_group_cq_id", + Description: "Unique CloudQuery ID of aws_cloudwatchlogs_log_groups table (FK)", + Type: schema.TypeUUID, + Resolver: schema.ParentIdResolver, + }, + { + Name: "arn", + Description: "The ARN (Amazon Resource Name) for the stream.", + Type: schema.TypeString, + }, + { + Name: "creation_time", + Description: "The creation time of the log group, expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC.", + Type: schema.TypeBigInt, + }, + { + Name: "first_event_timestamp", + Description: "The time of the first event, expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC.", + Type: schema.TypeBigInt, + }, + { + Name: "last_event_timestamp", + Description: "The time of the most recent log event in the log stream in CloudWatch Logs. This number is expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC. ", + Type: schema.TypeBigInt, + }, + { + Name: "last_ingestion_time", + Description: "The ingestion time, expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC.", + Type: schema.TypeBigInt, + }, + { + Name: "log_stream_name", + Description: "The name of the CloudWatch metric.", + Type: schema.TypeString, + }, + { + Name: "stored_bytes", + Description: "The number of bytes stored.", + Type: schema.TypeBigInt, + }, + }, + }, + }, } } @@ -133,7 +149,27 @@ func fetchCloudwatchlogsLogGroups(ctx context.Context, meta schema.ClientMeta, p return nil } -// func fetchCloudwatchlogsLogGroupsTransformations(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- interface{}) error { -// res <- parent.Item.(types.LogGroup) -// return nil -// } +func fetchCloudwatchlogsLogStreams(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- interface{}) error { + r := parent.Item.(types.LogGroup) + config := cloudwatchlogs.DescribeLogStreamsInput{ + LogGroupName: r.LogGroupName, + } + + c := meta.(*client.Client) + svc := c.Services().CloudwatchLogs + + for { + response, err := svc.DescribeLogStreams(ctx, &config, func(options *cloudwatchlogs.Options) { + options.Region = c.Region + }) + if err != nil { + return diag.WrapError(err) + } + res <- response.LogStreams + if aws.ToString(response.NextToken) == "" { + break + } + config.NextToken = response.NextToken + } + return nil +} From e5b85dddb3de4c858516b425abbd9213428f7df3 Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Wed, 27 Jul 2022 14:51:04 -0400 Subject: [PATCH 3/9] Disabling Log Streams due to record size. --- .../aws_cloudwatchlogs_log_group_streams.md | 14 -- .../services/cloudwatchlogs/log_groups.go | 147 +++++++++--------- 2 files changed, 73 insertions(+), 88 deletions(-) delete mode 100644 docs/tables/aws_cloudwatchlogs_log_group_streams.md diff --git a/docs/tables/aws_cloudwatchlogs_log_group_streams.md b/docs/tables/aws_cloudwatchlogs_log_group_streams.md deleted file mode 100644 index 710be4dcd..000000000 --- a/docs/tables/aws_cloudwatchlogs_log_group_streams.md +++ /dev/null @@ -1,14 +0,0 @@ - -# Table: aws_cloudwatchlogs_log_group_streams -The Log Streams connected to a particular Log Group, each of which is a sequence of log events from a single emitter of logs -## Columns -| Name | Type | Description | -| ------------- | ------------- | ----- | -|log_group_cq_id|uuid|Unique CloudQuery ID of aws_cloudwatchlogs_log_groups table (FK)| -|arn|text|The ARN (Amazon Resource Name) for the stream.| -|creation_time|bigint|The creation time of the log group, expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC.| -|first_event_timestamp|bigint|The time of the first event, expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC.| -|last_event_timestamp|bigint|The time of the most recent log event in the log stream in CloudWatch Logs. This number is expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC. | -|last_ingestion_time|bigint|The ingestion time, expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC.| -|log_stream_name|text|The name of the CloudWatch metric.| -|stored_bytes|bigint|The number of bytes stored.| diff --git a/resources/services/cloudwatchlogs/log_groups.go b/resources/services/cloudwatchlogs/log_groups.go index 85be240de..a43980bdc 100644 --- a/resources/services/cloudwatchlogs/log_groups.go +++ b/resources/services/cloudwatchlogs/log_groups.go @@ -5,7 +5,6 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs" - "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs/types" "github.com/cloudquery/cq-provider-aws/client" "github.com/cloudquery/cq-provider-sdk/provider/diag" "github.com/cloudquery/cq-provider-sdk/provider/schema" @@ -72,57 +71,57 @@ func CloudwatchlogsLogGroups() *schema.Table { Type: schema.TypeBigInt, }, }, - Relations: []*schema.Table{ - { - Name: "aws_cloudwatchlogs_log_group_streams", - Description: "The Log Streams connected to a particular Log Group, each of which is a sequence of log events from a single emitter of logs", - Resolver: fetchCloudwatchlogsLogStreams, - IgnoreInTests: true, - Columns: []schema.Column{ - { - Name: "log_group_cq_id", - Description: "Unique CloudQuery ID of aws_cloudwatchlogs_log_groups table (FK)", - Type: schema.TypeUUID, - Resolver: schema.ParentIdResolver, - }, - { - Name: "arn", - Description: "The ARN (Amazon Resource Name) for the stream.", - Type: schema.TypeString, - }, - { - Name: "creation_time", - Description: "The creation time of the log group, expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC.", - Type: schema.TypeBigInt, - }, - { - Name: "first_event_timestamp", - Description: "The time of the first event, expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC.", - Type: schema.TypeBigInt, - }, - { - Name: "last_event_timestamp", - Description: "The time of the most recent log event in the log stream in CloudWatch Logs. This number is expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC. ", - Type: schema.TypeBigInt, - }, - { - Name: "last_ingestion_time", - Description: "The ingestion time, expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC.", - Type: schema.TypeBigInt, - }, - { - Name: "log_stream_name", - Description: "The name of the CloudWatch metric.", - Type: schema.TypeString, - }, - { - Name: "stored_bytes", - Description: "The number of bytes stored.", - Type: schema.TypeBigInt, - }, - }, - }, - }, + // Relations: []*schema.Table{ + // { + // Name: "aws_cloudwatchlogs_log_group_streams", + // Description: "The Log Streams connected to a particular Log Group, each of which is a sequence of log events from a single emitter of logs", + // Resolver: fetchCloudwatchlogsLogStreams, + // IgnoreInTests: true, + // Columns: []schema.Column{ + // { + // Name: "log_group_cq_id", + // Description: "Unique CloudQuery ID of aws_cloudwatchlogs_log_groups table (FK)", + // Type: schema.TypeUUID, + // Resolver: schema.ParentIdResolver, + // }, + // { + // Name: "arn", + // Description: "The ARN (Amazon Resource Name) for the stream.", + // Type: schema.TypeString, + // }, + // { + // Name: "creation_time", + // Description: "The creation time of the log group, expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC.", + // Type: schema.TypeBigInt, + // }, + // { + // Name: "first_event_timestamp", + // Description: "The time of the first event, expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC.", + // Type: schema.TypeBigInt, + // }, + // { + // Name: "last_event_timestamp", + // Description: "The time of the most recent log event in the log stream in CloudWatch Logs. This number is expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC. ", + // Type: schema.TypeBigInt, + // }, + // { + // Name: "last_ingestion_time", + // Description: "The ingestion time, expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC.", + // Type: schema.TypeBigInt, + // }, + // { + // Name: "log_stream_name", + // Description: "The name of the CloudWatch metric.", + // Type: schema.TypeString, + // }, + // { + // Name: "stored_bytes", + // Description: "The number of bytes stored.", + // Type: schema.TypeBigInt, + // }, + // }, + // }, + // }, } } @@ -149,27 +148,27 @@ func fetchCloudwatchlogsLogGroups(ctx context.Context, meta schema.ClientMeta, p return nil } -func fetchCloudwatchlogsLogStreams(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- interface{}) error { - r := parent.Item.(types.LogGroup) - config := cloudwatchlogs.DescribeLogStreamsInput{ - LogGroupName: r.LogGroupName, - } +// func fetchCloudwatchlogsLogStreams(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- interface{}) error { +// r := parent.Item.(types.LogGroup) +// config := cloudwatchlogs.DescribeLogStreamsInput{ +// LogGroupName: r.LogGroupName, +// } - c := meta.(*client.Client) - svc := c.Services().CloudwatchLogs +// c := meta.(*client.Client) +// svc := c.Services().CloudwatchLogs - for { - response, err := svc.DescribeLogStreams(ctx, &config, func(options *cloudwatchlogs.Options) { - options.Region = c.Region - }) - if err != nil { - return diag.WrapError(err) - } - res <- response.LogStreams - if aws.ToString(response.NextToken) == "" { - break - } - config.NextToken = response.NextToken - } - return nil -} +// for { +// response, err := svc.DescribeLogStreams(ctx, &config, func(options *cloudwatchlogs.Options) { +// options.Region = c.Region +// }) +// if err != nil { +// return diag.WrapError(err) +// } +// res <- response.LogStreams +// if aws.ToString(response.NextToken) == "" { +// break +// } +// config.NextToken = response.NextToken +// } +// return nil +// } From 42f4eb6ef8bdc9ec0c64346f8386423a8bbf9301 Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Thu, 28 Jul 2022 10:24:39 -0400 Subject: [PATCH 4/9] Setting up config for cq-gen. --- docs/tables/aws_cloudwatchlogs_log_groups.md | 12 +- resources/provider/provider.go | 2 +- resources/services/cloudwatchlogs/gen.hcl | 41 ++++++ .../services/cloudwatchlogs/log_groups.go | 118 ++++-------------- 4 files changed, 69 insertions(+), 104 deletions(-) create mode 100644 resources/services/cloudwatchlogs/gen.hcl diff --git a/docs/tables/aws_cloudwatchlogs_log_groups.md b/docs/tables/aws_cloudwatchlogs_log_groups.md index 4f3178c62..70e570369 100644 --- a/docs/tables/aws_cloudwatchlogs_log_groups.md +++ b/docs/tables/aws_cloudwatchlogs_log_groups.md @@ -1,15 +1,15 @@ # Table: aws_cloudwatchlogs_log_groups -CloudWatch Logs enables you to centralize the logs from all of your systems, applications, and AWS services that you use, in a single, highly scalable service. Log groups define groups of log streams that share the same retention, monitoring, and access control settings. Each log stream has to belong to one log group. +Represents a log group. ## Columns | Name | Type | Description | | ------------- | ------------- | ----- | |account_id|text|The AWS Account ID of the resource.| |region|text|The AWS Region of the resource.| -|arn|text|The ARN (Amazon Resource Name) for the group.| +|arn|text|The Amazon Resource Name (ARN) of the log group.| |creation_time|bigint|The creation time of the log group, expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC.| |kms_key_id|text|The Amazon Resource Name (ARN) of the CMK to use when encrypting log data.| -|name|text|The name of the log group.| -|metric_filter_count|bigint|The number of metric filters| -|retention_in_days|bigint|The number of days to retain the log events in the specified log group. Possible values are: 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, 2192, 2557, 2922, 3288, and 3653.| -|stored_bytes|bigint|The number of bytes stored| +|log_group_name|text|The name of the log group.| +|metric_filter_count|bigint|The number of metric filters.| +|retention_in_days|bigint|The number of days to retain the log events in the specified log group| +|stored_bytes|bigint|The number of bytes stored.| diff --git a/resources/provider/provider.go b/resources/provider/provider.go index fd9d19dcb..f1cc64722 100644 --- a/resources/provider/provider.go +++ b/resources/provider/provider.go @@ -109,7 +109,7 @@ func Provider() *provider.Provider { "cloudtrail.trails": cloudtrail.CloudtrailTrails(), "cloudwatch.alarms": cloudwatch.CloudwatchAlarms(), "cloudwatchlogs.filters": cloudwatchlogs.CloudwatchlogsFilters(), - "cloudwatchlogs.log_groups": cloudwatchlogs.CloudwatchlogsLogGroups(), + "cloudwatchlogs.log_groups": cloudwatchlogs.LogGroups(), "codebuild.projects": codebuild.CodebuildProjects(), "codepipeline.pipelines": codepipeline.Pipelines(), "codepipeline.webhooks": codepipeline.Webhooks(), diff --git a/resources/services/cloudwatchlogs/gen.hcl b/resources/services/cloudwatchlogs/gen.hcl new file mode 100644 index 000000000..16eef5670 --- /dev/null +++ b/resources/services/cloudwatchlogs/gen.hcl @@ -0,0 +1,41 @@ +resource "aws" "cloudwatchlogs" "log_groups" { + path = "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs/types.LogGroup" + ignoreError "IgnoreAccessDenied" { + path = "github.com/cloudquery/cq-provider-aws/client.IgnoreAccessDeniedServiceDisabled" + } + deleteFilter "AccountRegionFilter" { + path = "github.com/cloudquery/cq-provider-aws/client.DeleteAccountRegionFilter" + } + multiplex "AwsAccountRegion" { + path = "github.com/cloudquery/cq-provider-aws/client.ServiceAccountRegionMultiplexer" + params = ["logs"] + } + + userDefinedColumn "account_id" { + description = "The AWS Account ID of the resource." + type = "string" + resolver "resolveAWSAccount" { + path = "github.com/cloudquery/cq-provider-aws/client.ResolveAWSAccount" + } + } + + userDefinedColumn "region" { + type = "string" + description = "The AWS Region of the resource." + resolver "resolveAWSRegion" { + path = "github.com/cloudquery/cq-provider-aws/client.ResolveAWSRegion" + } + } + ignore_columns_in_tests = ["kms_key_id","retention_in_days"] + + options { + primary_keys = ["arn"] + } + + column "tags" { + type = "json" + resolver "resolveTags" { + path = "github.com/cloudquery/cq-provider-aws/client.ResolveTags" + } + } +} \ No newline at end of file diff --git a/resources/services/cloudwatchlogs/log_groups.go b/resources/services/cloudwatchlogs/log_groups.go index a43980bdc..3f95b62cc 100644 --- a/resources/services/cloudwatchlogs/log_groups.go +++ b/resources/services/cloudwatchlogs/log_groups.go @@ -10,16 +10,15 @@ import ( "github.com/cloudquery/cq-provider-sdk/provider/schema" ) -func CloudwatchlogsLogGroups() *schema.Table { +func LogGroups() *schema.Table { return &schema.Table{ - Name: "aws_cloudwatchlogs_log_groups", - Description: "CloudWatch Logs enables you to centralize the logs from all of your systems, applications, and AWS services that you use, in a single, highly scalable service. Log groups define groups of log streams that share the same retention, monitoring, and access control settings. Each log stream has to belong to one log group. ", - Resolver: fetchCloudwatchlogsLogGroups, - Multiplex: client.ServiceAccountRegionMultiplexer("logs"), - IgnoreError: client.IgnoreAccessDeniedServiceDisabled, - DeleteFilter: client.DeleteAccountRegionFilter, - Options: schema.TableCreationOptions{PrimaryKeys: []string{"account_id", "region", "name"}}, - IgnoreInTests: true, + Name: "aws_cloudwatchlogs_log_groups", + Description: "Represents a log group.", + Resolver: fetchCloudwatchlogsLogGroups, + Multiplex: client.ServiceAccountRegionMultiplexer("logs"), + IgnoreError: client.IgnoreAccessDeniedServiceDisabled, + DeleteFilter: client.DeleteAccountRegionFilter, + Options: schema.TableCreationOptions{PrimaryKeys: []string{"arn"}}, Columns: []schema.Column{ { Name: "account_id", @@ -35,7 +34,7 @@ func CloudwatchlogsLogGroups() *schema.Table { }, { Name: "arn", - Description: "The ARN (Amazon Resource Name) for the group.", + Description: "The Amazon Resource Name (ARN) of the log group.", Type: schema.TypeString, }, { @@ -44,90 +43,40 @@ func CloudwatchlogsLogGroups() *schema.Table { Type: schema.TypeBigInt, }, { - Name: "kms_key_id", - Description: "The Amazon Resource Name (ARN) of the CMK to use when encrypting log data.", - Type: schema.TypeString, - Resolver: schema.PathResolver("KmsKeyId"), + Name: "kms_key_id", + Description: "The Amazon Resource Name (ARN) of the CMK to use when encrypting log data.", + Type: schema.TypeString, + IgnoreInTests: true, }, { - Name: "name", + Name: "log_group_name", Description: "The name of the log group.", Type: schema.TypeString, - Resolver: schema.PathResolver("LogGroupName"), }, { Name: "metric_filter_count", - Description: "The number of metric filters", + Description: "The number of metric filters.", Type: schema.TypeBigInt, }, { - Name: "retention_in_days", - Description: "The number of days to retain the log events in the specified log group. Possible values are: 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, 2192, 2557, 2922, 3288, and 3653.", - Type: schema.TypeBigInt, + Name: "retention_in_days", + Description: "The number of days to retain the log events in the specified log group", + Type: schema.TypeBigInt, + IgnoreInTests: true, }, { Name: "stored_bytes", - Description: "The number of bytes stored", + Description: "The number of bytes stored.", Type: schema.TypeBigInt, }, }, - // Relations: []*schema.Table{ - // { - // Name: "aws_cloudwatchlogs_log_group_streams", - // Description: "The Log Streams connected to a particular Log Group, each of which is a sequence of log events from a single emitter of logs", - // Resolver: fetchCloudwatchlogsLogStreams, - // IgnoreInTests: true, - // Columns: []schema.Column{ - // { - // Name: "log_group_cq_id", - // Description: "Unique CloudQuery ID of aws_cloudwatchlogs_log_groups table (FK)", - // Type: schema.TypeUUID, - // Resolver: schema.ParentIdResolver, - // }, - // { - // Name: "arn", - // Description: "The ARN (Amazon Resource Name) for the stream.", - // Type: schema.TypeString, - // }, - // { - // Name: "creation_time", - // Description: "The creation time of the log group, expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC.", - // Type: schema.TypeBigInt, - // }, - // { - // Name: "first_event_timestamp", - // Description: "The time of the first event, expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC.", - // Type: schema.TypeBigInt, - // }, - // { - // Name: "last_event_timestamp", - // Description: "The time of the most recent log event in the log stream in CloudWatch Logs. This number is expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC. ", - // Type: schema.TypeBigInt, - // }, - // { - // Name: "last_ingestion_time", - // Description: "The ingestion time, expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC.", - // Type: schema.TypeBigInt, - // }, - // { - // Name: "log_stream_name", - // Description: "The name of the CloudWatch metric.", - // Type: schema.TypeString, - // }, - // { - // Name: "stored_bytes", - // Description: "The number of bytes stored.", - // Type: schema.TypeBigInt, - // }, - // }, - // }, - // }, } } // ==================================================================================================================== // Table Resolver Functions // ==================================================================================================================== + func fetchCloudwatchlogsLogGroups(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- interface{}) error { var config cloudwatchlogs.DescribeLogGroupsInput c := meta.(*client.Client) @@ -147,28 +96,3 @@ func fetchCloudwatchlogsLogGroups(ctx context.Context, meta schema.ClientMeta, p } return nil } - -// func fetchCloudwatchlogsLogStreams(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- interface{}) error { -// r := parent.Item.(types.LogGroup) -// config := cloudwatchlogs.DescribeLogStreamsInput{ -// LogGroupName: r.LogGroupName, -// } - -// c := meta.(*client.Client) -// svc := c.Services().CloudwatchLogs - -// for { -// response, err := svc.DescribeLogStreams(ctx, &config, func(options *cloudwatchlogs.Options) { -// options.Region = c.Region -// }) -// if err != nil { -// return diag.WrapError(err) -// } -// res <- response.LogStreams -// if aws.ToString(response.NextToken) == "" { -// break -// } -// config.NextToken = response.NextToken -// } -// return nil -// } From e437fd975b80f632a53f16decdf132d9b83f7139 Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Thu, 28 Jul 2022 10:56:52 -0400 Subject: [PATCH 5/9] Missed function rename in test. --- resources/services/cloudwatchlogs/log_groups_mock_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/services/cloudwatchlogs/log_groups_mock_test.go b/resources/services/cloudwatchlogs/log_groups_mock_test.go index 6f3129236..7b619aca8 100644 --- a/resources/services/cloudwatchlogs/log_groups_mock_test.go +++ b/resources/services/cloudwatchlogs/log_groups_mock_test.go @@ -28,5 +28,5 @@ func buildCloudwatchLogsLogGroupsMock(t *testing.T, ctrl *gomock.Controller) cli } func TestCloudwatchlogsLogGroups(t *testing.T) { - client.AwsMockTestHelper(t, CloudwatchlogsLogGroups(), buildCloudwatchLogsLogGroupsMock, client.TestOptions{}) + client.AwsMockTestHelper(t, LogGroups(), buildCloudwatchLogsLogGroupsMock, client.TestOptions{}) } From 203f1ad3acb76b4efaa0e7fb1f6142a56ead6944 Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Fri, 29 Jul 2022 14:52:37 -0400 Subject: [PATCH 6/9] Log Groups: adding tag resolver. --- client/mocks/mock_cloudwatchlogs.go | 20 +++++++++++++++++++ client/services.go | 1 + docs/tables/aws_cloudwatchlogs_log_groups.md | 1 + resources/services/cloudwatchlogs/gen.hcl | 12 +++++------ .../services/cloudwatchlogs/log_groups.go | 17 ++++++++++++++++ 5 files changed, 45 insertions(+), 6 deletions(-) diff --git a/client/mocks/mock_cloudwatchlogs.go b/client/mocks/mock_cloudwatchlogs.go index da6515fc3..9ac917eeb 100644 --- a/client/mocks/mock_cloudwatchlogs.go +++ b/client/mocks/mock_cloudwatchlogs.go @@ -94,3 +94,23 @@ func (mr *MockCloudwatchLogsClientMockRecorder) DescribeMetricFilters(arg0, arg1 varargs := append([]interface{}{arg0, arg1}, arg2...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeMetricFilters", reflect.TypeOf((*MockCloudwatchLogsClient)(nil).DescribeMetricFilters), varargs...) } + +// ListTagsLogGroup mocks base method. +func (m *MockCloudwatchLogsClient) ListTagsLogGroup(arg0 context.Context, arg1 *cloudwatchlogs.ListTagsLogGroupInput, arg2 ...func(*cloudwatchlogs.Options)) (*cloudwatchlogs.ListTagsLogGroupOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ListTagsLogGroup", varargs...) + ret0, _ := ret[0].(*cloudwatchlogs.ListTagsLogGroupOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListTagsLogGroup indicates an expected call of ListTagsLogGroup. +func (mr *MockCloudwatchLogsClientMockRecorder) ListTagsLogGroup(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListTagsLogGroup", reflect.TypeOf((*MockCloudwatchLogsClient)(nil).ListTagsLogGroup), varargs...) +} diff --git a/client/services.go b/client/services.go index 681a5b4c2..e3d06aa30 100644 --- a/client/services.go +++ b/client/services.go @@ -174,6 +174,7 @@ type CloudwatchLogsClient interface { DescribeMetricFilters(ctx context.Context, params *cloudwatchlogs.DescribeMetricFiltersInput, optFns ...func(*cloudwatchlogs.Options)) (*cloudwatchlogs.DescribeMetricFiltersOutput, error) DescribeLogGroups(ctx context.Context, params *cloudwatchlogs.DescribeLogGroupsInput, optFns ...func(*cloudwatchlogs.Options)) (*cloudwatchlogs.DescribeLogGroupsOutput, error) DescribeLogStreams(ctx context.Context, params *cloudwatchlogs.DescribeLogStreamsInput, optFns ...func(*cloudwatchlogs.Options)) (*cloudwatchlogs.DescribeLogStreamsOutput, error) + ListTagsLogGroup(ctx context.Context, params *cloudwatchlogs.ListTagsLogGroupInput, optFns ...func(*cloudwatchlogs.Options)) (*cloudwatchlogs.ListTagsLogGroupOutput, error) } //go:generate mockgen -package=mocks -destination=./mocks/mock_cloudformation.go . CloudFormationClient diff --git a/docs/tables/aws_cloudwatchlogs_log_groups.md b/docs/tables/aws_cloudwatchlogs_log_groups.md index 70e570369..46e64b0b6 100644 --- a/docs/tables/aws_cloudwatchlogs_log_groups.md +++ b/docs/tables/aws_cloudwatchlogs_log_groups.md @@ -6,6 +6,7 @@ Represents a log group. | ------------- | ------------- | ----- | |account_id|text|The AWS Account ID of the resource.| |region|text|The AWS Region of the resource.| +|tags|jsonb|| |arn|text|The Amazon Resource Name (ARN) of the log group.| |creation_time|bigint|The creation time of the log group, expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC.| |kms_key_id|text|The Amazon Resource Name (ARN) of the CMK to use when encrypting log data.| diff --git a/resources/services/cloudwatchlogs/gen.hcl b/resources/services/cloudwatchlogs/gen.hcl index 16eef5670..66f5b91c9 100644 --- a/resources/services/cloudwatchlogs/gen.hcl +++ b/resources/services/cloudwatchlogs/gen.hcl @@ -26,16 +26,16 @@ resource "aws" "cloudwatchlogs" "log_groups" { path = "github.com/cloudquery/cq-provider-aws/client.ResolveAWSRegion" } } + + userDefinedColumn "tags" { + type = "json" + generate_resolver = true + } + ignore_columns_in_tests = ["kms_key_id","retention_in_days"] options { primary_keys = ["arn"] } - column "tags" { - type = "json" - resolver "resolveTags" { - path = "github.com/cloudquery/cq-provider-aws/client.ResolveTags" - } - } } \ No newline at end of file diff --git a/resources/services/cloudwatchlogs/log_groups.go b/resources/services/cloudwatchlogs/log_groups.go index 3f95b62cc..b59c1abef 100644 --- a/resources/services/cloudwatchlogs/log_groups.go +++ b/resources/services/cloudwatchlogs/log_groups.go @@ -5,6 +5,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs" + "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs/types" "github.com/cloudquery/cq-provider-aws/client" "github.com/cloudquery/cq-provider-sdk/provider/diag" "github.com/cloudquery/cq-provider-sdk/provider/schema" @@ -32,6 +33,11 @@ func LogGroups() *schema.Table { Type: schema.TypeString, Resolver: client.ResolveAWSRegion, }, + { + Name: "tags", + Type: schema.TypeJSON, + Resolver: ResolveCloudwatchlogsLogGroupTags, + }, { Name: "arn", Description: "The Amazon Resource Name (ARN) of the log group.", @@ -96,3 +102,14 @@ func fetchCloudwatchlogsLogGroups(ctx context.Context, meta schema.ClientMeta, p } return nil } + +func ResolveCloudwatchlogsLogGroupTags(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { + lg := resource.Item.(types.LogGroup) + cl := meta.(*client.Client) + svc := cl.Services().CloudwatchLogs + out, err := svc.ListTagsLogGroup(ctx, &cloudwatchlogs.ListTagsLogGroupInput{LogGroupName: lg.LogGroupName}) + if err != nil { + return diag.WrapError(err) + } + return diag.WrapError(resource.Set(c.Name, out.Tags)) +} From 3247adcc1cfa50f382bd3aada776c7581adf6d15 Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Tue, 2 Aug 2022 08:39:15 -0400 Subject: [PATCH 7/9] Adding tags description and go:generate command. --- docs/tables/aws_cloudwatchlogs_log_groups.md | 2 +- resources/services/cloudwatchlogs/gen.hcl | 1 + resources/services/cloudwatchlogs/log_groups.go | 9 +++++---- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/tables/aws_cloudwatchlogs_log_groups.md b/docs/tables/aws_cloudwatchlogs_log_groups.md index 46e64b0b6..5145c50c9 100644 --- a/docs/tables/aws_cloudwatchlogs_log_groups.md +++ b/docs/tables/aws_cloudwatchlogs_log_groups.md @@ -6,7 +6,7 @@ Represents a log group. | ------------- | ------------- | ----- | |account_id|text|The AWS Account ID of the resource.| |region|text|The AWS Region of the resource.| -|tags|jsonb|| +|tags|jsonb|The tags for the log group.| |arn|text|The Amazon Resource Name (ARN) of the log group.| |creation_time|bigint|The creation time of the log group, expressed as the number of milliseconds after Jan 1, 1970 00:00:00 UTC.| |kms_key_id|text|The Amazon Resource Name (ARN) of the CMK to use when encrypting log data.| diff --git a/resources/services/cloudwatchlogs/gen.hcl b/resources/services/cloudwatchlogs/gen.hcl index 66f5b91c9..895887d4a 100644 --- a/resources/services/cloudwatchlogs/gen.hcl +++ b/resources/services/cloudwatchlogs/gen.hcl @@ -30,6 +30,7 @@ resource "aws" "cloudwatchlogs" "log_groups" { userDefinedColumn "tags" { type = "json" generate_resolver = true + description = "The tags for the log group." } ignore_columns_in_tests = ["kms_key_id","retention_in_days"] diff --git a/resources/services/cloudwatchlogs/log_groups.go b/resources/services/cloudwatchlogs/log_groups.go index b59c1abef..43e0bb396 100644 --- a/resources/services/cloudwatchlogs/log_groups.go +++ b/resources/services/cloudwatchlogs/log_groups.go @@ -11,6 +11,7 @@ import ( "github.com/cloudquery/cq-provider-sdk/provider/schema" ) +//go:generate cq-gen --resource=log_groups --config gen.hcl --output . func LogGroups() *schema.Table { return &schema.Table{ Name: "aws_cloudwatchlogs_log_groups", @@ -34,9 +35,10 @@ func LogGroups() *schema.Table { Resolver: client.ResolveAWSRegion, }, { - Name: "tags", - Type: schema.TypeJSON, - Resolver: ResolveCloudwatchlogsLogGroupTags, + Name: "tags", + Description: "The tags for the log group.", + Type: schema.TypeJSON, + Resolver: ResolveCloudwatchlogsLogGroupTags, }, { Name: "arn", @@ -102,7 +104,6 @@ func fetchCloudwatchlogsLogGroups(ctx context.Context, meta schema.ClientMeta, p } return nil } - func ResolveCloudwatchlogsLogGroupTags(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { lg := resource.Item.(types.LogGroup) cl := meta.(*client.Client) From fe648bc4d0c9523d7c7ed510fbf2181cd3cac806 Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Tue, 2 Aug 2022 10:55:44 -0400 Subject: [PATCH 8/9] Cleanup from unsupported LogStreams endpoint. --- client/mocks/mock_cloudwatchlogs.go | 20 -------------------- client/services.go | 1 - 2 files changed, 21 deletions(-) diff --git a/client/mocks/mock_cloudwatchlogs.go b/client/mocks/mock_cloudwatchlogs.go index 9ac917eeb..0ec2d5162 100644 --- a/client/mocks/mock_cloudwatchlogs.go +++ b/client/mocks/mock_cloudwatchlogs.go @@ -55,26 +55,6 @@ func (mr *MockCloudwatchLogsClientMockRecorder) DescribeLogGroups(arg0, arg1 int return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeLogGroups", reflect.TypeOf((*MockCloudwatchLogsClient)(nil).DescribeLogGroups), varargs...) } -// DescribeLogStreams mocks base method. -func (m *MockCloudwatchLogsClient) DescribeLogStreams(arg0 context.Context, arg1 *cloudwatchlogs.DescribeLogStreamsInput, arg2 ...func(*cloudwatchlogs.Options)) (*cloudwatchlogs.DescribeLogStreamsOutput, error) { - m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "DescribeLogStreams", varargs...) - ret0, _ := ret[0].(*cloudwatchlogs.DescribeLogStreamsOutput) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// DescribeLogStreams indicates an expected call of DescribeLogStreams. -func (mr *MockCloudwatchLogsClientMockRecorder) DescribeLogStreams(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DescribeLogStreams", reflect.TypeOf((*MockCloudwatchLogsClient)(nil).DescribeLogStreams), varargs...) -} - // DescribeMetricFilters mocks base method. func (m *MockCloudwatchLogsClient) DescribeMetricFilters(arg0 context.Context, arg1 *cloudwatchlogs.DescribeMetricFiltersInput, arg2 ...func(*cloudwatchlogs.Options)) (*cloudwatchlogs.DescribeMetricFiltersOutput, error) { m.ctrl.T.Helper() diff --git a/client/services.go b/client/services.go index e3d06aa30..456e56cd6 100644 --- a/client/services.go +++ b/client/services.go @@ -173,7 +173,6 @@ type CloudwatchClient interface { type CloudwatchLogsClient interface { DescribeMetricFilters(ctx context.Context, params *cloudwatchlogs.DescribeMetricFiltersInput, optFns ...func(*cloudwatchlogs.Options)) (*cloudwatchlogs.DescribeMetricFiltersOutput, error) DescribeLogGroups(ctx context.Context, params *cloudwatchlogs.DescribeLogGroupsInput, optFns ...func(*cloudwatchlogs.Options)) (*cloudwatchlogs.DescribeLogGroupsOutput, error) - DescribeLogStreams(ctx context.Context, params *cloudwatchlogs.DescribeLogStreamsInput, optFns ...func(*cloudwatchlogs.Options)) (*cloudwatchlogs.DescribeLogStreamsOutput, error) ListTagsLogGroup(ctx context.Context, params *cloudwatchlogs.ListTagsLogGroupInput, optFns ...func(*cloudwatchlogs.Options)) (*cloudwatchlogs.ListTagsLogGroupOutput, error) } From 07acbf67030c2cefa18f8cb89630985df2975164 Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Wed, 3 Aug 2022 09:53:02 -0400 Subject: [PATCH 9/9] Adding tests and cq-gen config. --- resources/services/cloudwatchlogs/gen.hcl | 4 ++++ resources/services/cloudwatchlogs/log_groups.go | 2 +- .../services/cloudwatchlogs/log_groups_mock_test.go | 9 +++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/resources/services/cloudwatchlogs/gen.hcl b/resources/services/cloudwatchlogs/gen.hcl index 895887d4a..9d8a17445 100644 --- a/resources/services/cloudwatchlogs/gen.hcl +++ b/resources/services/cloudwatchlogs/gen.hcl @@ -1,3 +1,7 @@ +service = "aws" +output_directory = "." +add_generate = true + resource "aws" "cloudwatchlogs" "log_groups" { path = "github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs/types.LogGroup" ignoreError "IgnoreAccessDenied" { diff --git a/resources/services/cloudwatchlogs/log_groups.go b/resources/services/cloudwatchlogs/log_groups.go index 43e0bb396..35230c6ca 100644 --- a/resources/services/cloudwatchlogs/log_groups.go +++ b/resources/services/cloudwatchlogs/log_groups.go @@ -11,7 +11,7 @@ import ( "github.com/cloudquery/cq-provider-sdk/provider/schema" ) -//go:generate cq-gen --resource=log_groups --config gen.hcl --output . +//go:generate cq-gen --resource log_groups --config gen.hcl --output . func LogGroups() *schema.Table { return &schema.Table{ Name: "aws_cloudwatchlogs_log_groups", diff --git a/resources/services/cloudwatchlogs/log_groups_mock_test.go b/resources/services/cloudwatchlogs/log_groups_mock_test.go index 7b619aca8..a9e75fcf3 100644 --- a/resources/services/cloudwatchlogs/log_groups_mock_test.go +++ b/resources/services/cloudwatchlogs/log_groups_mock_test.go @@ -22,6 +22,15 @@ func buildCloudwatchLogsLogGroupsMock(t *testing.T, ctrl *gomock.Controller) cli &cloudwatchlogs.DescribeLogGroupsOutput{ LogGroups: []types.LogGroup{l}, }, nil) + + tags := &cloudwatchlogs.ListTagsLogGroupOutput{} + err = faker.FakeData(tags) + if err != nil { + t.Fatal(err) + } + + m.EXPECT().ListTagsLogGroup(gomock.Any(), gomock.Any(), gomock.Any()).Return(tags, nil) + return client.Services{ CloudwatchLogs: m, }