From 597782cef346c15f6be73c0cb3cace4d2981068c Mon Sep 17 00:00:00 2001 From: Timur Irmatov Date: Thu, 28 Jul 2022 18:17:35 +0300 Subject: [PATCH 1/8] feat: Implement Glue registries resource --- client/mocks/glue.go | 60 +++++ client/services.go | 3 + docs/tables/aws_glue_registries.md | 15 ++ docs/tables/aws_glue_registry_schemas.md | 21 ++ resources/provider/provider.go | 1 + resources/services/glue/registries.go | 244 +++++++++++++++++++++ resources/services/glue/registries.hcl | 54 +++++ resources/services/glue/registries_test.go | 70 ++++++ 8 files changed, 468 insertions(+) create mode 100644 docs/tables/aws_glue_registries.md create mode 100644 docs/tables/aws_glue_registry_schemas.md create mode 100644 resources/services/glue/registries.go create mode 100644 resources/services/glue/registries.hcl create mode 100644 resources/services/glue/registries_test.go diff --git a/client/mocks/glue.go b/client/mocks/glue.go index 3c3fe3a70..71db53016 100644 --- a/client/mocks/glue.go +++ b/client/mocks/glue.go @@ -35,6 +35,26 @@ func (m *MockGlueClient) EXPECT() *MockGlueClientMockRecorder { return m.recorder } +// GetSchema mocks base method. +func (m *MockGlueClient) GetSchema(arg0 context.Context, arg1 *glue.GetSchemaInput, arg2 ...func(*glue.Options)) (*glue.GetSchemaOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetSchema", varargs...) + ret0, _ := ret[0].(*glue.GetSchemaOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetSchema indicates an expected call of GetSchema. +func (mr *MockGlueClientMockRecorder) GetSchema(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSchema", reflect.TypeOf((*MockGlueClient)(nil).GetSchema), varargs...) +} + // GetTags mocks base method. func (m *MockGlueClient) GetTags(arg0 context.Context, arg1 *glue.GetTagsInput, arg2 ...func(*glue.Options)) (*glue.GetTagsOutput, error) { m.ctrl.T.Helper() @@ -75,6 +95,46 @@ func (mr *MockGlueClientMockRecorder) GetWorkflow(arg0, arg1 interface{}, arg2 . return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetWorkflow", reflect.TypeOf((*MockGlueClient)(nil).GetWorkflow), varargs...) } +// ListRegistries mocks base method. +func (m *MockGlueClient) ListRegistries(arg0 context.Context, arg1 *glue.ListRegistriesInput, arg2 ...func(*glue.Options)) (*glue.ListRegistriesOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ListRegistries", varargs...) + ret0, _ := ret[0].(*glue.ListRegistriesOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListRegistries indicates an expected call of ListRegistries. +func (mr *MockGlueClientMockRecorder) ListRegistries(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRegistries", reflect.TypeOf((*MockGlueClient)(nil).ListRegistries), varargs...) +} + +// ListSchemas mocks base method. +func (m *MockGlueClient) ListSchemas(arg0 context.Context, arg1 *glue.ListSchemasInput, arg2 ...func(*glue.Options)) (*glue.ListSchemasOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ListSchemas", varargs...) + ret0, _ := ret[0].(*glue.ListSchemasOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListSchemas indicates an expected call of ListSchemas. +func (mr *MockGlueClientMockRecorder) ListSchemas(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListSchemas", reflect.TypeOf((*MockGlueClient)(nil).ListSchemas), varargs...) +} + // ListWorkflows mocks base method. func (m *MockGlueClient) ListWorkflows(arg0 context.Context, arg1 *glue.ListWorkflowsInput, arg2 ...func(*glue.Options)) (*glue.ListWorkflowsOutput, error) { m.ctrl.T.Helper() diff --git a/client/services.go b/client/services.go index 082131cdd..19693d396 100644 --- a/client/services.go +++ b/client/services.go @@ -748,7 +748,10 @@ type SESClient interface { //go:generate mockgen -package=mocks -destination=./mocks/glue.go . GlueClient type GlueClient interface { + GetSchema(ctx context.Context, params *glue.GetSchemaInput, optFns ...func(*glue.Options)) (*glue.GetSchemaOutput, error) GetTags(ctx context.Context, params *glue.GetTagsInput, optFns ...func(*glue.Options)) (*glue.GetTagsOutput, error) GetWorkflow(ctx context.Context, params *glue.GetWorkflowInput, optFns ...func(*glue.Options)) (*glue.GetWorkflowOutput, error) + ListRegistries(ctx context.Context, params *glue.ListRegistriesInput, optFns ...func(*glue.Options)) (*glue.ListRegistriesOutput, error) + ListSchemas(ctx context.Context, params *glue.ListSchemasInput, optFns ...func(*glue.Options)) (*glue.ListSchemasOutput, error) ListWorkflows(ctx context.Context, params *glue.ListWorkflowsInput, optFns ...func(*glue.Options)) (*glue.ListWorkflowsOutput, error) } diff --git a/docs/tables/aws_glue_registries.md b/docs/tables/aws_glue_registries.md new file mode 100644 index 000000000..072abb920 --- /dev/null +++ b/docs/tables/aws_glue_registries.md @@ -0,0 +1,15 @@ + +# Table: aws_glue_registries +A structure containing the details for a registry. +## Columns +| Name | Type | Description | +| ------------- | ------------- | ----- | +|account_id|text|The AWS Account ID of the resource.| +|region|text|The AWS Region of the resource.| +|tags|jsonb|Resource tags.| +|created_time|text|The data the registry was created.| +|description|text|A description of the registry.| +|arn|text|The Amazon Resource Name (ARN) of the registry.| +|registry_name|text|The name of the registry.| +|status|text|The status of the registry.| +|updated_time|text|The date the registry was updated.| diff --git a/docs/tables/aws_glue_registry_schemas.md b/docs/tables/aws_glue_registry_schemas.md new file mode 100644 index 000000000..c3465eb36 --- /dev/null +++ b/docs/tables/aws_glue_registry_schemas.md @@ -0,0 +1,21 @@ + +# Table: aws_glue_registry_schemas + +## Columns +| Name | Type | Description | +| ------------- | ------------- | ----- | +|registry_cq_id|uuid|Unique CloudQuery ID of aws_glue_registries table (FK)| +|tags|jsonb|Resource tags.| +|compatibility|text|The compatibility mode of the schema.| +|created_time|text|The date and time the schema was created.| +|data_format|text|The data format of the schema definition| +|description|text|A description of schema if specified when created| +|latest_schema_version|bigint|The latest version of the schema associated with the returned schema definition.| +|next_schema_version|bigint|The next version of the schema associated with the returned schema definition.| +|registry_arn|text|The Amazon Resource Name (ARN) of the registry.| +|registry_name|text|The name of the registry.| +|arn|text|The Amazon Resource Name (ARN) of the schema.| +|schema_checkpoint|bigint|The version number of the checkpoint (the last time the compatibility mode was changed).| +|schema_name|text|The name of the schema.| +|schema_status|text|The status of the schema.| +|updated_time|text|The date and time the schema was updated.| diff --git a/resources/provider/provider.go b/resources/provider/provider.go index c30312e94..55f67221c 100644 --- a/resources/provider/provider.go +++ b/resources/provider/provider.go @@ -167,6 +167,7 @@ func Provider() *provider.Provider { "emr.block_public_access_configs": emr.EmrBlockPublicAccessConfigs(), "emr.clusters": emr.EmrClusters(), "fsx.backups": fsx.FsxBackups(), + "glue.registries": glue.Registries(), "glue.workflows": glue.Workflows(), "guardduty.detectors": guardduty.GuarddutyDetectors(), "iam.accounts": iam.IamAccounts(), diff --git a/resources/services/glue/registries.go b/resources/services/glue/registries.go new file mode 100644 index 000000000..d0784b798 --- /dev/null +++ b/resources/services/glue/registries.go @@ -0,0 +1,244 @@ +package glue + +import ( + "context" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/glue" + "github.com/aws/aws-sdk-go-v2/service/glue/types" + "github.com/cloudquery/cq-provider-aws/client" + "github.com/cloudquery/cq-provider-sdk/provider/diag" + "github.com/cloudquery/cq-provider-sdk/provider/schema" +) + +//go:generate cq-gen --resource registries --config registries.hcl --output . +func Registries() *schema.Table { + return &schema.Table{ + Name: "aws_glue_registries", + Description: "A structure containing the details for a registry.", + Resolver: fetchGlueRegistries, + Multiplex: client.ServiceAccountRegionMultiplexer("glue"), + IgnoreError: client.IgnoreAccessDeniedServiceDisabled, + DeleteFilter: client.DeleteAccountRegionFilter, + Options: schema.TableCreationOptions{PrimaryKeys: []string{"arn"}}, + Columns: []schema.Column{ + { + Name: "account_id", + Description: "The AWS Account ID of the resource.", + Type: schema.TypeString, + Resolver: client.ResolveAWSAccount, + }, + { + Name: "region", + Description: "The AWS Region of the resource.", + Type: schema.TypeString, + Resolver: client.ResolveAWSRegion, + }, + { + Name: "tags", + Description: "Resource tags.", + Type: schema.TypeJSON, + Resolver: resolveGlueRegistryTags, + }, + { + Name: "created_time", + Description: "The data the registry was created.", + Type: schema.TypeString, + }, + { + Name: "description", + Description: "A description of the registry.", + Type: schema.TypeString, + }, + { + Name: "arn", + Description: "The Amazon Resource Name (ARN) of the registry.", + Type: schema.TypeString, + Resolver: schema.PathResolver("RegistryArn"), + }, + { + Name: "registry_name", + Description: "The name of the registry.", + Type: schema.TypeString, + }, + { + Name: "status", + Description: "The status of the registry.", + Type: schema.TypeString, + }, + { + Name: "updated_time", + Description: "The date the registry was updated.", + Type: schema.TypeString, + }, + }, + Relations: []*schema.Table{ + { + Name: "aws_glue_registry_schemas", + Resolver: fetchGlueRegistrySchemas, + Columns: []schema.Column{ + { + Name: "registry_cq_id", + Description: "Unique CloudQuery ID of aws_glue_registries table (FK)", + Type: schema.TypeUUID, + Resolver: schema.ParentIdResolver, + }, + { + Name: "tags", + Description: "Resource tags.", + Type: schema.TypeJSON, + Resolver: resolveGlueRegistrySchemaTags, + }, + { + Name: "compatibility", + Description: "The compatibility mode of the schema.", + Type: schema.TypeString, + }, + { + Name: "created_time", + Description: "The date and time the schema was created.", + Type: schema.TypeString, + }, + { + Name: "data_format", + Description: "The data format of the schema definition", + Type: schema.TypeString, + }, + { + Name: "description", + Description: "A description of schema if specified when created", + Type: schema.TypeString, + }, + { + Name: "latest_schema_version", + Description: "The latest version of the schema associated with the returned schema definition.", + Type: schema.TypeBigInt, + }, + { + Name: "next_schema_version", + Description: "The next version of the schema associated with the returned schema definition.", + Type: schema.TypeBigInt, + }, + { + Name: "registry_arn", + Description: "The Amazon Resource Name (ARN) of the registry.", + Type: schema.TypeString, + }, + { + Name: "registry_name", + Description: "The name of the registry.", + Type: schema.TypeString, + }, + { + Name: "arn", + Description: "The Amazon Resource Name (ARN) of the schema.", + Type: schema.TypeString, + Resolver: schema.PathResolver("SchemaArn"), + }, + { + Name: "schema_checkpoint", + Description: "The version number of the checkpoint (the last time the compatibility mode was changed).", + Type: schema.TypeBigInt, + }, + { + Name: "schema_name", + Description: "The name of the schema.", + Type: schema.TypeString, + }, + { + Name: "schema_status", + Description: "The status of the schema.", + Type: schema.TypeString, + }, + { + Name: "updated_time", + Description: "The date and time the schema was updated.", + Type: schema.TypeString, + }, + }, + }, + }, + } +} + +// ==================================================================================================================== +// Table Resolver Functions +// ==================================================================================================================== + +func fetchGlueRegistries(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- interface{}) error { + cl := meta.(*client.Client) + svc := cl.Services().Glue + input := glue.ListRegistriesInput{MaxResults: aws.Int32(100)} + for { + result, err := svc.ListRegistries(ctx, &input) + if err != nil { + return diag.WrapError(err) + } + res <- result.Registries + if aws.ToString(result.NextToken) == "" { + break + } + input.NextToken = result.NextToken + } + return nil +} +func resolveGlueRegistryTags(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { + cl := meta.(*client.Client) + svc := cl.Services().Glue + r := resource.Item.(types.RegistryListItem) + result, err := svc.GetTags(ctx, &glue.GetTagsInput{ + ResourceArn: r.RegistryArn, + }) + if err != nil { + if cl.IsNotFoundError(err) { + return nil + } + return diag.WrapError(err) + } + return diag.WrapError(resource.Set(c.Name, result.Tags)) +} +func fetchGlueRegistrySchemas(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- interface{}) error { + r := parent.Item.(types.RegistryListItem) + cl := meta.(*client.Client) + svc := cl.Services().Glue + input := glue.ListSchemasInput{ + RegistryId: &types.RegistryId{RegistryArn: r.RegistryArn}, + MaxResults: aws.Int32(100), + } + for { + result, err := svc.ListSchemas(ctx, &input) + if err != nil { + return diag.WrapError(err) + } + for _, item := range result.Schemas { + s, err := svc.GetSchema(ctx, &glue.GetSchemaInput{SchemaId: &types.SchemaId{SchemaArn: item.SchemaArn}}) + if err != nil { + if cl.IsNotFoundError(err) { + continue + } + return diag.WrapError(err) + } + res <- s + } + if aws.ToString(result.NextToken) == "" { + break + } + input.NextToken = result.NextToken + } + return nil +} +func resolveGlueRegistrySchemaTags(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { + cl := meta.(*client.Client) + svc := cl.Services().Glue + s := resource.Item.(*glue.GetSchemaOutput) + result, err := svc.GetTags(ctx, &glue.GetTagsInput{ + ResourceArn: s.SchemaArn, + }) + if err != nil { + if cl.IsNotFoundError(err) { + return nil + } + return diag.WrapError(err) + } + return diag.WrapError(resource.Set(c.Name, result.Tags)) +} diff --git a/resources/services/glue/registries.hcl b/resources/services/glue/registries.hcl new file mode 100644 index 000000000..a3bba046a --- /dev/null +++ b/resources/services/glue/registries.hcl @@ -0,0 +1,54 @@ +service = "aws" +output_directory = "." +add_generate = true + +resource "aws" "glue" "registries" { + path = "github.com/aws/aws-sdk-go-v2/service/glue/types.RegistryListItem" + ignoreError "IgnoreAccessDenied" { + path = "github.com/cloudquery/cq-provider-aws/client.IgnoreAccessDeniedServiceDisabled" + } + deleteFilter "AccountRegionFilter" { + path = "github.com/cloudquery/cq-provider-aws/client.DeleteAccountRegionFilter" + } + multiplex "AwsAccountRegion" { + path = "github.com/cloudquery/cq-provider-aws/client.ServiceAccountRegionMultiplexer" + params = ["glue"] + } + options { + primary_keys = ["arn"] + } + column "registry_arn" { + rename = "arn" + } + userDefinedColumn "account_id" { + description = "The AWS Account ID of the resource." + type = "string" + resolver "resolveAWSAccount" { + path = "github.com/cloudquery/cq-provider-aws/client.ResolveAWSAccount" + } + } + userDefinedColumn "region" { + type = "string" + description = "The AWS Region of the resource." + resolver "resolveAWSRegion" { + path = "github.com/cloudquery/cq-provider-aws/client.ResolveAWSRegion" + } + } + + userDefinedColumn "tags" { + type = "json" + description = "Resource tags." + generate_resolver = true + } + user_relation "aws" "glue" "schemas" { + path = "github.com/aws/aws-sdk-go-v2/service/glue.GetSchemaOutput" + userDefinedColumn "tags" { + type = "json" + description = "Resource tags." + generate_resolver = true + } + column "schema_arn" { + rename = "arn" + } + } +} diff --git a/resources/services/glue/registries_test.go b/resources/services/glue/registries_test.go new file mode 100644 index 000000000..750e8642d --- /dev/null +++ b/resources/services/glue/registries_test.go @@ -0,0 +1,70 @@ +package glue + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/glue" + "github.com/aws/aws-sdk-go-v2/service/glue/types" + "github.com/cloudquery/cq-provider-aws/client" + "github.com/cloudquery/cq-provider-aws/client/mocks" + "github.com/cloudquery/faker/v3" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" +) + +func buildRegistriesMock(t *testing.T, ctrl *gomock.Controller) client.Services { + m := mocks.NewMockGlueClient(ctrl) + + var r types.RegistryListItem + require.NoError(t, faker.FakeData(&r)) + m.EXPECT().ListRegistries( + gomock.Any(), + &glue.ListRegistriesInput{MaxResults: aws.Int32(100)}, + ).Return( + &glue.ListRegistriesOutput{Registries: []types.RegistryListItem{r}}, + nil, + ) + + m.EXPECT().GetTags( + gomock.Any(), + &glue.GetTagsInput{ResourceArn: r.RegistryArn}, + ).Return( + &glue.GetTagsOutput{Tags: map[string]string{"tag": "value"}}, + nil, + ) + + var s glue.GetSchemaOutput + require.NoError(t, faker.FakeData(&s)) + m.EXPECT().ListSchemas( + gomock.Any(), + &glue.ListSchemasInput{ + RegistryId: &types.RegistryId{RegistryArn: r.RegistryArn}, + MaxResults: aws.Int32(100), + }, + ).Return( + &glue.ListSchemasOutput{Schemas: []types.SchemaListItem{{SchemaArn: s.SchemaArn}}}, + nil, + ) + + m.EXPECT().GetSchema( + gomock.Any(), + &glue.GetSchemaInput{SchemaId: &types.SchemaId{SchemaArn: s.SchemaArn}}, + ).Return(&s, nil) + + m.EXPECT().GetTags( + gomock.Any(), + &glue.GetTagsInput{ResourceArn: s.SchemaArn}, + ).Return( + &glue.GetTagsOutput{Tags: map[string]string{"tag": "value"}}, + nil, + ) + + return client.Services{ + Glue: m, + } +} + +func TestRegistries(t *testing.T) { + client.AwsMockTestHelper(t, Registries(), buildRegistriesMock, client.TestOptions{}) +} From 0d7c5f1e99c9218e8915406dbbc00005572c1ba1 Mon Sep 17 00:00:00 2001 From: amanenk Date: Wed, 3 Aug 2022 14:46:23 +0300 Subject: [PATCH 2/8] fix --- client/mocks/glue.go | 60 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/client/mocks/glue.go b/client/mocks/glue.go index a6691cba9..2ba15fa1d 100644 --- a/client/mocks/glue.go +++ b/client/mocks/glue.go @@ -155,6 +155,26 @@ func (mr *MockGlueClientMockRecorder) GetMLTransforms(arg0, arg1 interface{}, ar return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMLTransforms", reflect.TypeOf((*MockGlueClient)(nil).GetMLTransforms), varargs...) } +// GetSchema mocks base method. +func (m *MockGlueClient) GetSchema(arg0 context.Context, arg1 *glue.GetSchemaInput, arg2 ...func(*glue.Options)) (*glue.GetSchemaOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetSchema", varargs...) + ret0, _ := ret[0].(*glue.GetSchemaOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetSchema indicates an expected call of GetSchema. +func (mr *MockGlueClientMockRecorder) GetSchema(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSchema", reflect.TypeOf((*MockGlueClient)(nil).GetSchema), varargs...) +} + // GetTables mocks base method. func (m *MockGlueClient) GetTables(arg0 context.Context, arg1 *glue.GetTablesInput, arg2 ...func(*glue.Options)) (*glue.GetTablesOutput, error) { m.ctrl.T.Helper() @@ -215,6 +235,46 @@ func (mr *MockGlueClientMockRecorder) GetWorkflow(arg0, arg1 interface{}, arg2 . return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetWorkflow", reflect.TypeOf((*MockGlueClient)(nil).GetWorkflow), varargs...) } +// ListRegistries mocks base method. +func (m *MockGlueClient) ListRegistries(arg0 context.Context, arg1 *glue.ListRegistriesInput, arg2 ...func(*glue.Options)) (*glue.ListRegistriesOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ListRegistries", varargs...) + ret0, _ := ret[0].(*glue.ListRegistriesOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListRegistries indicates an expected call of ListRegistries. +func (mr *MockGlueClientMockRecorder) ListRegistries(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRegistries", reflect.TypeOf((*MockGlueClient)(nil).ListRegistries), varargs...) +} + +// ListSchemas mocks base method. +func (m *MockGlueClient) ListSchemas(arg0 context.Context, arg1 *glue.ListSchemasInput, arg2 ...func(*glue.Options)) (*glue.ListSchemasOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ListSchemas", varargs...) + ret0, _ := ret[0].(*glue.ListSchemasOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListSchemas indicates an expected call of ListSchemas. +func (mr *MockGlueClientMockRecorder) ListSchemas(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListSchemas", reflect.TypeOf((*MockGlueClient)(nil).ListSchemas), varargs...) +} + // ListWorkflows mocks base method. func (m *MockGlueClient) ListWorkflows(arg0 context.Context, arg1 *glue.ListWorkflowsInput, arg2 ...func(*glue.Options)) (*glue.ListWorkflowsOutput, error) { m.ctrl.T.Helper() From 49fd256d9259ae5193df512796622d42d360bfa4 Mon Sep 17 00:00:00 2001 From: amanenk Date: Wed, 3 Aug 2022 15:32:10 +0300 Subject: [PATCH 3/8] added schema versions added version metadata --- client/mocks/glue.go | 60 ++++++++++ client/services.go | 3 + resources/services/glue/registries.go | 127 +++++++++++++++++++++ resources/services/glue/registries.hcl | 20 +++- resources/services/glue/registries_test.go | 23 ++++ 5 files changed, 229 insertions(+), 4 deletions(-) diff --git a/client/mocks/glue.go b/client/mocks/glue.go index 2ba15fa1d..d68b35bcd 100644 --- a/client/mocks/glue.go +++ b/client/mocks/glue.go @@ -175,6 +175,26 @@ func (mr *MockGlueClientMockRecorder) GetSchema(arg0, arg1 interface{}, arg2 ... return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSchema", reflect.TypeOf((*MockGlueClient)(nil).GetSchema), varargs...) } +// GetSchemaVersion mocks base method. +func (m *MockGlueClient) GetSchemaVersion(arg0 context.Context, arg1 *glue.GetSchemaVersionInput, arg2 ...func(*glue.Options)) (*glue.GetSchemaVersionOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetSchemaVersion", varargs...) + ret0, _ := ret[0].(*glue.GetSchemaVersionOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetSchemaVersion indicates an expected call of GetSchemaVersion. +func (mr *MockGlueClientMockRecorder) GetSchemaVersion(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSchemaVersion", reflect.TypeOf((*MockGlueClient)(nil).GetSchemaVersion), varargs...) +} + // GetTables mocks base method. func (m *MockGlueClient) GetTables(arg0 context.Context, arg1 *glue.GetTablesInput, arg2 ...func(*glue.Options)) (*glue.GetTablesOutput, error) { m.ctrl.T.Helper() @@ -255,6 +275,26 @@ func (mr *MockGlueClientMockRecorder) ListRegistries(arg0, arg1 interface{}, arg return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRegistries", reflect.TypeOf((*MockGlueClient)(nil).ListRegistries), varargs...) } +// ListSchemaVersions mocks base method. +func (m *MockGlueClient) ListSchemaVersions(arg0 context.Context, arg1 *glue.ListSchemaVersionsInput, arg2 ...func(*glue.Options)) (*glue.ListSchemaVersionsOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ListSchemaVersions", varargs...) + ret0, _ := ret[0].(*glue.ListSchemaVersionsOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListSchemaVersions indicates an expected call of ListSchemaVersions. +func (mr *MockGlueClientMockRecorder) ListSchemaVersions(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListSchemaVersions", reflect.TypeOf((*MockGlueClient)(nil).ListSchemaVersions), varargs...) +} + // ListSchemas mocks base method. func (m *MockGlueClient) ListSchemas(arg0 context.Context, arg1 *glue.ListSchemasInput, arg2 ...func(*glue.Options)) (*glue.ListSchemasOutput, error) { m.ctrl.T.Helper() @@ -294,3 +334,23 @@ func (mr *MockGlueClientMockRecorder) ListWorkflows(arg0, arg1 interface{}, arg2 varargs := append([]interface{}{arg0, arg1}, arg2...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListWorkflows", reflect.TypeOf((*MockGlueClient)(nil).ListWorkflows), varargs...) } + +// QuerySchemaVersionMetadata mocks base method. +func (m *MockGlueClient) QuerySchemaVersionMetadata(arg0 context.Context, arg1 *glue.QuerySchemaVersionMetadataInput, arg2 ...func(*glue.Options)) (*glue.QuerySchemaVersionMetadataOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "QuerySchemaVersionMetadata", varargs...) + ret0, _ := ret[0].(*glue.QuerySchemaVersionMetadataOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// QuerySchemaVersionMetadata indicates an expected call of QuerySchemaVersionMetadata. +func (mr *MockGlueClientMockRecorder) QuerySchemaVersionMetadata(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "QuerySchemaVersionMetadata", reflect.TypeOf((*MockGlueClient)(nil).QuerySchemaVersionMetadata), varargs...) +} diff --git a/client/services.go b/client/services.go index 50f583e5e..ca0a0f7ed 100644 --- a/client/services.go +++ b/client/services.go @@ -764,6 +764,9 @@ type GlueClient interface { GetMLTransforms(ctx context.Context, params *glue.GetMLTransformsInput, optFns ...func(*glue.Options)) (*glue.GetMLTransformsOutput, error) GetMLTaskRuns(ctx context.Context, params *glue.GetMLTaskRunsInput, optFns ...func(*glue.Options)) (*glue.GetMLTaskRunsOutput, error) GetDataCatalogEncryptionSettings(ctx context.Context, params *glue.GetDataCatalogEncryptionSettingsInput, optFns ...func(*glue.Options)) (*glue.GetDataCatalogEncryptionSettingsOutput, error) + ListSchemaVersions(ctx context.Context, params *glue.ListSchemaVersionsInput, optFns ...func(*glue.Options)) (*glue.ListSchemaVersionsOutput, error) + GetSchemaVersion(ctx context.Context, params *glue.GetSchemaVersionInput, optFns ...func(*glue.Options)) (*glue.GetSchemaVersionOutput, error) + QuerySchemaVersionMetadata(ctx context.Context, params *glue.QuerySchemaVersionMetadataInput, optFns ...func(*glue.Options)) (*glue.QuerySchemaVersionMetadataOutput, error) } //go:generate mockgen -package=mocks -destination=./mocks/kinesis.go . KinesisClient diff --git a/resources/services/glue/registries.go b/resources/services/glue/registries.go index d0784b798..778c74b0f 100644 --- a/resources/services/glue/registries.go +++ b/resources/services/glue/registries.go @@ -156,6 +156,61 @@ func Registries() *schema.Table { Type: schema.TypeString, }, }, + Relations: []*schema.Table{ + { + Name: "aws_glue_registry_schema_versions", + Resolver: fetchGlueRegistrySchemaVersions, + Columns: []schema.Column{ + { + Name: "registry_schema_cq_id", + Description: "Unique CloudQuery ID of aws_glue_registry_schemas table (FK)", + Type: schema.TypeUUID, + Resolver: schema.ParentIdResolver, + }, + { + Name: "metadata", + Type: schema.TypeJSON, + Resolver: resolveGlueRegistrySchemaVersionMetadata, + }, + { + Name: "created_time", + Description: "The date and time the schema version was created.", + Type: schema.TypeString, + }, + { + Name: "data_format", + Description: "The data format of the schema definition", + Type: schema.TypeString, + }, + { + Name: "schema_arn", + Description: "The Amazon Resource Name (ARN) of the schema.", + Type: schema.TypeString, + }, + { + Name: "schema_definition", + Description: "The schema definition for the schema ID.", + Type: schema.TypeString, + }, + { + Name: "id", + Description: "The SchemaVersionId of the schema version.", + Type: schema.TypeString, + Resolver: schema.PathResolver("SchemaVersionId"), + }, + { + Name: "status", + Description: "The status of the schema version.", + Type: schema.TypeString, + }, + { + Name: "version_number", + Description: "The version number of the schema.", + Type: schema.TypeBigInt, + }, + }, + }, + }, }, }, } @@ -242,3 +297,75 @@ func resolveGlueRegistrySchemaTags(ctx context.Context, meta schema.ClientMeta, } return diag.WrapError(resource.Set(c.Name, result.Tags)) } +func fetchGlueRegistrySchemaVersions(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- interface{}) error { + cl := meta.(*client.Client) + s := parent.Item.(*glue.GetSchemaOutput) + svc := cl.Services().Glue + schemaId := types.SchemaId{ + SchemaArn: s.SchemaArn, + } + input := glue.ListSchemaVersionsInput{ + SchemaId: &schemaId, + MaxResults: aws.Int32(100), + } + for { + result, err := svc.ListSchemaVersions(ctx, &input) + if err != nil { + return diag.WrapError(err) + } + for _, item := range result.Schemas { + s, err := svc.GetSchemaVersion(ctx, &glue.GetSchemaVersionInput{ + SchemaId: &schemaId, + SchemaVersionId: item.SchemaVersionId, + SchemaVersionNumber: &types.SchemaVersionNumber{ + VersionNumber: item.VersionNumber, + }}) + if err != nil { + if cl.IsNotFoundError(err) { + continue + } + return diag.WrapError(err) + } + res <- s + } + if aws.ToString(result.NextToken) == "" { + break + } + input.NextToken = result.NextToken + } + return nil +} +func resolveGlueRegistrySchemaVersionMetadata(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { + cl := meta.(*client.Client) + svc := cl.Services().Glue + s := resource.Item.(*glue.GetSchemaVersionOutput) + input := &glue.QuerySchemaVersionMetadataInput{ + SchemaId: &types.SchemaId{ + SchemaArn: s.SchemaArn, + }, + SchemaVersionId: s.SchemaVersionId, + SchemaVersionNumber: &types.SchemaVersionNumber{ + VersionNumber: s.VersionNumber, + }, + } + metadata := make(map[string]types.MetadataInfo) + for { + result, err := svc.QuerySchemaVersionMetadata(ctx, input) + if err != nil { + if cl.IsNotFoundError(err) { + return nil + } + return diag.WrapError(err) + } + + for k, v := range result.MetadataInfoMap { + metadata[k] = v + } + + if aws.ToString(result.NextToken) == "" { + break + } + input.NextToken = result.NextToken + } + return diag.WrapError(resource.Set(c.Name, metadata)) +} diff --git a/resources/services/glue/registries.hcl b/resources/services/glue/registries.hcl index a3bba046a..40db70c1c 100644 --- a/resources/services/glue/registries.hcl +++ b/resources/services/glue/registries.hcl @@ -36,19 +36,31 @@ resource "aws" "glue" "registries" { } userDefinedColumn "tags" { - type = "json" - description = "Resource tags." + type = "json" + description = "Resource tags." generate_resolver = true } user_relation "aws" "glue" "schemas" { path = "github.com/aws/aws-sdk-go-v2/service/glue.GetSchemaOutput" userDefinedColumn "tags" { - type = "json" - description = "Resource tags." + type = "json" + description = "Resource tags." generate_resolver = true } column "schema_arn" { rename = "arn" } + + user_relation "aws" "glue" "versions" { + path = "github.com/aws/aws-sdk-go-v2/service/glue.GetSchemaVersionOutput" + userDefinedColumn "metadata" { + type = "json" + generate_resolver = true + } + + column "schema_version_id" { + rename = "id" + } + } } } diff --git a/resources/services/glue/registries_test.go b/resources/services/glue/registries_test.go index 750e8642d..969d832d6 100644 --- a/resources/services/glue/registries_test.go +++ b/resources/services/glue/registries_test.go @@ -60,6 +60,29 @@ func buildRegistriesMock(t *testing.T, ctrl *gomock.Controller) client.Services nil, ) + var lsv glue.ListSchemaVersionsOutput + require.NoError(t, faker.FakeData(&lsv)) + lsv.NextToken = nil + m.EXPECT().ListSchemaVersions( + gomock.Any(), + gomock.Any(), + ).Return(&lsv, nil) + + var sv glue.GetSchemaVersionOutput + require.NoError(t, faker.FakeData(&sv)) + m.EXPECT().GetSchemaVersion( + gomock.Any(), + gomock.Any(), + ).Return(&sv, nil) + + var sm glue.QuerySchemaVersionMetadataOutput + require.NoError(t, faker.FakeData(&sm)) + sm.NextToken = nil + m.EXPECT().QuerySchemaVersionMetadata( + gomock.Any(), + gomock.Any(), + ).Return(&sm, nil) + return client.Services{ Glue: m, } From dd068f0627a273374a1cda828be35630853995c2 Mon Sep 17 00:00:00 2001 From: amanenk Date: Wed, 3 Aug 2022 15:40:48 +0300 Subject: [PATCH 4/8] docs --- docs/tables/aws_glue_registry_schema_versions.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 docs/tables/aws_glue_registry_schema_versions.md diff --git a/docs/tables/aws_glue_registry_schema_versions.md b/docs/tables/aws_glue_registry_schema_versions.md new file mode 100644 index 000000000..a48048dae --- /dev/null +++ b/docs/tables/aws_glue_registry_schema_versions.md @@ -0,0 +1,15 @@ + +# Table: aws_glue_registry_schema_versions + +## Columns +| Name | Type | Description | +| ------------- | ------------- | ----- | +|registry_schema_cq_id|uuid|Unique CloudQuery ID of aws_glue_registry_schemas table (FK)| +|metadata|jsonb|| +|created_time|text|The date and time the schema version was created.| +|data_format|text|The data format of the schema definition| +|schema_arn|text|The Amazon Resource Name (ARN) of the schema.| +|schema_definition|text|The schema definition for the schema ID.| +|id|text|The SchemaVersionId of the schema version.| +|status|text|The status of the schema version.| +|version_number|bigint|The version number of the schema.| From 33b6d2c710f66f757f725973c757f476df74fc05 Mon Sep 17 00:00:00 2001 From: amanenk Date: Wed, 3 Aug 2022 18:23:58 +0300 Subject: [PATCH 5/8] mock --- client/mocks/glue.go | 120 +++++++++++++++++++++++++++++++++++++++++++ client/services.go | 1 + 2 files changed, 121 insertions(+) diff --git a/client/mocks/glue.go b/client/mocks/glue.go index 4df3fbd64..fa4385ea1 100644 --- a/client/mocks/glue.go +++ b/client/mocks/glue.go @@ -175,6 +175,46 @@ func (mr *MockGlueClientMockRecorder) GetMLTransforms(arg0, arg1 interface{}, ar return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMLTransforms", reflect.TypeOf((*MockGlueClient)(nil).GetMLTransforms), varargs...) } +// GetSchema mocks base method. +func (m *MockGlueClient) GetSchema(arg0 context.Context, arg1 *glue.GetSchemaInput, arg2 ...func(*glue.Options)) (*glue.GetSchemaOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetSchema", varargs...) + ret0, _ := ret[0].(*glue.GetSchemaOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetSchema indicates an expected call of GetSchema. +func (mr *MockGlueClientMockRecorder) GetSchema(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSchema", reflect.TypeOf((*MockGlueClient)(nil).GetSchema), varargs...) +} + +// GetSchemaVersion mocks base method. +func (m *MockGlueClient) GetSchemaVersion(arg0 context.Context, arg1 *glue.GetSchemaVersionInput, arg2 ...func(*glue.Options)) (*glue.GetSchemaVersionOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetSchemaVersion", varargs...) + ret0, _ := ret[0].(*glue.GetSchemaVersionOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetSchemaVersion indicates an expected call of GetSchemaVersion. +func (mr *MockGlueClientMockRecorder) GetSchemaVersion(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSchemaVersion", reflect.TypeOf((*MockGlueClient)(nil).GetSchemaVersion), varargs...) +} + // GetTables mocks base method. func (m *MockGlueClient) GetTables(arg0 context.Context, arg1 *glue.GetTablesInput, arg2 ...func(*glue.Options)) (*glue.GetTablesOutput, error) { m.ctrl.T.Helper() @@ -255,6 +295,66 @@ func (mr *MockGlueClientMockRecorder) GetWorkflow(arg0, arg1 interface{}, arg2 . return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetWorkflow", reflect.TypeOf((*MockGlueClient)(nil).GetWorkflow), varargs...) } +// ListRegistries mocks base method. +func (m *MockGlueClient) ListRegistries(arg0 context.Context, arg1 *glue.ListRegistriesInput, arg2 ...func(*glue.Options)) (*glue.ListRegistriesOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ListRegistries", varargs...) + ret0, _ := ret[0].(*glue.ListRegistriesOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListRegistries indicates an expected call of ListRegistries. +func (mr *MockGlueClientMockRecorder) ListRegistries(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRegistries", reflect.TypeOf((*MockGlueClient)(nil).ListRegistries), varargs...) +} + +// ListSchemaVersions mocks base method. +func (m *MockGlueClient) ListSchemaVersions(arg0 context.Context, arg1 *glue.ListSchemaVersionsInput, arg2 ...func(*glue.Options)) (*glue.ListSchemaVersionsOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ListSchemaVersions", varargs...) + ret0, _ := ret[0].(*glue.ListSchemaVersionsOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListSchemaVersions indicates an expected call of ListSchemaVersions. +func (mr *MockGlueClientMockRecorder) ListSchemaVersions(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListSchemaVersions", reflect.TypeOf((*MockGlueClient)(nil).ListSchemaVersions), varargs...) +} + +// ListSchemas mocks base method. +func (m *MockGlueClient) ListSchemas(arg0 context.Context, arg1 *glue.ListSchemasInput, arg2 ...func(*glue.Options)) (*glue.ListSchemasOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ListSchemas", varargs...) + ret0, _ := ret[0].(*glue.ListSchemasOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListSchemas indicates an expected call of ListSchemas. +func (mr *MockGlueClientMockRecorder) ListSchemas(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListSchemas", reflect.TypeOf((*MockGlueClient)(nil).ListSchemas), varargs...) +} + // ListTriggers mocks base method. func (m *MockGlueClient) ListTriggers(arg0 context.Context, arg1 *glue.ListTriggersInput, arg2 ...func(*glue.Options)) (*glue.ListTriggersOutput, error) { m.ctrl.T.Helper() @@ -294,3 +394,23 @@ func (mr *MockGlueClientMockRecorder) ListWorkflows(arg0, arg1 interface{}, arg2 varargs := append([]interface{}{arg0, arg1}, arg2...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListWorkflows", reflect.TypeOf((*MockGlueClient)(nil).ListWorkflows), varargs...) } + +// QuerySchemaVersionMetadata mocks base method. +func (m *MockGlueClient) QuerySchemaVersionMetadata(arg0 context.Context, arg1 *glue.QuerySchemaVersionMetadataInput, arg2 ...func(*glue.Options)) (*glue.QuerySchemaVersionMetadataOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "QuerySchemaVersionMetadata", varargs...) + ret0, _ := ret[0].(*glue.QuerySchemaVersionMetadataOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// QuerySchemaVersionMetadata indicates an expected call of QuerySchemaVersionMetadata. +func (mr *MockGlueClientMockRecorder) QuerySchemaVersionMetadata(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "QuerySchemaVersionMetadata", reflect.TypeOf((*MockGlueClient)(nil).QuerySchemaVersionMetadata), varargs...) +} diff --git a/client/services.go b/client/services.go index c7d386027..88ce3224a 100644 --- a/client/services.go +++ b/client/services.go @@ -765,6 +765,7 @@ type GlueClient interface { GetDataCatalogEncryptionSettings(ctx context.Context, params *glue.GetDataCatalogEncryptionSettingsInput, optFns ...func(*glue.Options)) (*glue.GetDataCatalogEncryptionSettingsOutput, error) ListRegistries(ctx context.Context, params *glue.ListRegistriesInput, optFns ...func(*glue.Options)) (*glue.ListRegistriesOutput, error) ListSchemas(ctx context.Context, params *glue.ListSchemasInput, optFns ...func(*glue.Options)) (*glue.ListSchemasOutput, error) + GetSchema(ctx context.Context, params *glue.GetSchemaInput, optFns ...func(*glue.Options)) (*glue.GetSchemaOutput, error) ListSchemaVersions(ctx context.Context, params *glue.ListSchemaVersionsInput, optFns ...func(*glue.Options)) (*glue.ListSchemaVersionsOutput, error) GetSchemaVersion(ctx context.Context, params *glue.GetSchemaVersionInput, optFns ...func(*glue.Options)) (*glue.GetSchemaVersionOutput, error) QuerySchemaVersionMetadata(ctx context.Context, params *glue.QuerySchemaVersionMetadataInput, optFns ...func(*glue.Options)) (*glue.QuerySchemaVersionMetadataOutput, error) From 027f99297b331eb31aef978341a94a993a4f7859 Mon Sep 17 00:00:00 2001 From: amanenk Date: Thu, 4 Aug 2022 12:30:54 +0300 Subject: [PATCH 6/8] fix --- resources/services/glue/registries.go | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/resources/services/glue/registries.go b/resources/services/glue/registries.go index 778c74b0f..64c658e40 100644 --- a/resources/services/glue/registries.go +++ b/resources/services/glue/registries.go @@ -315,11 +315,8 @@ func fetchGlueRegistrySchemaVersions(ctx context.Context, meta schema.ClientMeta } for _, item := range result.Schemas { s, err := svc.GetSchemaVersion(ctx, &glue.GetSchemaVersionInput{ - SchemaId: &schemaId, SchemaVersionId: item.SchemaVersionId, - SchemaVersionNumber: &types.SchemaVersionNumber{ - VersionNumber: item.VersionNumber, - }}) + }) if err != nil { if cl.IsNotFoundError(err) { continue @@ -340,13 +337,7 @@ func resolveGlueRegistrySchemaVersionMetadata(ctx context.Context, meta schema.C svc := cl.Services().Glue s := resource.Item.(*glue.GetSchemaVersionOutput) input := &glue.QuerySchemaVersionMetadataInput{ - SchemaId: &types.SchemaId{ - SchemaArn: s.SchemaArn, - }, SchemaVersionId: s.SchemaVersionId, - SchemaVersionNumber: &types.SchemaVersionNumber{ - VersionNumber: s.VersionNumber, - }, } metadata := make(map[string]types.MetadataInfo) for { From 655427875df720493be925e08162422f3f0ee30a Mon Sep 17 00:00:00 2001 From: amanenk Date: Thu, 4 Aug 2022 17:05:16 +0300 Subject: [PATCH 7/8] mock --- client/mocks/glue.go | 120 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/client/mocks/glue.go b/client/mocks/glue.go index 450ee3dea..64b722bce 100644 --- a/client/mocks/glue.go +++ b/client/mocks/glue.go @@ -195,6 +195,46 @@ func (mr *MockGlueClientMockRecorder) GetMLTransforms(arg0, arg1 interface{}, ar return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMLTransforms", reflect.TypeOf((*MockGlueClient)(nil).GetMLTransforms), varargs...) } +// GetSchema mocks base method. +func (m *MockGlueClient) GetSchema(arg0 context.Context, arg1 *glue.GetSchemaInput, arg2 ...func(*glue.Options)) (*glue.GetSchemaOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetSchema", varargs...) + ret0, _ := ret[0].(*glue.GetSchemaOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetSchema indicates an expected call of GetSchema. +func (mr *MockGlueClientMockRecorder) GetSchema(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSchema", reflect.TypeOf((*MockGlueClient)(nil).GetSchema), varargs...) +} + +// GetSchemaVersion mocks base method. +func (m *MockGlueClient) GetSchemaVersion(arg0 context.Context, arg1 *glue.GetSchemaVersionInput, arg2 ...func(*glue.Options)) (*glue.GetSchemaVersionOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetSchemaVersion", varargs...) + ret0, _ := ret[0].(*glue.GetSchemaVersionOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetSchemaVersion indicates an expected call of GetSchemaVersion. +func (mr *MockGlueClientMockRecorder) GetSchemaVersion(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSchemaVersion", reflect.TypeOf((*MockGlueClient)(nil).GetSchemaVersion), varargs...) +} + // GetSecurityConfigurations mocks base method. func (m *MockGlueClient) GetSecurityConfigurations(arg0 context.Context, arg1 *glue.GetSecurityConfigurationsInput, arg2 ...func(*glue.Options)) (*glue.GetSecurityConfigurationsOutput, error) { m.ctrl.T.Helper() @@ -295,6 +335,66 @@ func (mr *MockGlueClientMockRecorder) GetWorkflow(arg0, arg1 interface{}, arg2 . return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetWorkflow", reflect.TypeOf((*MockGlueClient)(nil).GetWorkflow), varargs...) } +// ListRegistries mocks base method. +func (m *MockGlueClient) ListRegistries(arg0 context.Context, arg1 *glue.ListRegistriesInput, arg2 ...func(*glue.Options)) (*glue.ListRegistriesOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ListRegistries", varargs...) + ret0, _ := ret[0].(*glue.ListRegistriesOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListRegistries indicates an expected call of ListRegistries. +func (mr *MockGlueClientMockRecorder) ListRegistries(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRegistries", reflect.TypeOf((*MockGlueClient)(nil).ListRegistries), varargs...) +} + +// ListSchemaVersions mocks base method. +func (m *MockGlueClient) ListSchemaVersions(arg0 context.Context, arg1 *glue.ListSchemaVersionsInput, arg2 ...func(*glue.Options)) (*glue.ListSchemaVersionsOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ListSchemaVersions", varargs...) + ret0, _ := ret[0].(*glue.ListSchemaVersionsOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListSchemaVersions indicates an expected call of ListSchemaVersions. +func (mr *MockGlueClientMockRecorder) ListSchemaVersions(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListSchemaVersions", reflect.TypeOf((*MockGlueClient)(nil).ListSchemaVersions), varargs...) +} + +// ListSchemas mocks base method. +func (m *MockGlueClient) ListSchemas(arg0 context.Context, arg1 *glue.ListSchemasInput, arg2 ...func(*glue.Options)) (*glue.ListSchemasOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ListSchemas", varargs...) + ret0, _ := ret[0].(*glue.ListSchemasOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListSchemas indicates an expected call of ListSchemas. +func (mr *MockGlueClientMockRecorder) ListSchemas(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListSchemas", reflect.TypeOf((*MockGlueClient)(nil).ListSchemas), varargs...) +} + // ListTriggers mocks base method. func (m *MockGlueClient) ListTriggers(arg0 context.Context, arg1 *glue.ListTriggersInput, arg2 ...func(*glue.Options)) (*glue.ListTriggersOutput, error) { m.ctrl.T.Helper() @@ -334,3 +434,23 @@ func (mr *MockGlueClientMockRecorder) ListWorkflows(arg0, arg1 interface{}, arg2 varargs := append([]interface{}{arg0, arg1}, arg2...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListWorkflows", reflect.TypeOf((*MockGlueClient)(nil).ListWorkflows), varargs...) } + +// QuerySchemaVersionMetadata mocks base method. +func (m *MockGlueClient) QuerySchemaVersionMetadata(arg0 context.Context, arg1 *glue.QuerySchemaVersionMetadataInput, arg2 ...func(*glue.Options)) (*glue.QuerySchemaVersionMetadataOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "QuerySchemaVersionMetadata", varargs...) + ret0, _ := ret[0].(*glue.QuerySchemaVersionMetadataOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// QuerySchemaVersionMetadata indicates an expected call of QuerySchemaVersionMetadata. +func (mr *MockGlueClientMockRecorder) QuerySchemaVersionMetadata(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "QuerySchemaVersionMetadata", reflect.TypeOf((*MockGlueClient)(nil).QuerySchemaVersionMetadata), varargs...) +} From fd07869fefaedaa0718af1b9282bbabf60eba6c4 Mon Sep 17 00:00:00 2001 From: amanenk Date: Thu, 4 Aug 2022 18:38:02 +0300 Subject: [PATCH 8/8] docs adjusted --- docs/tables/aws_glue_registry_schema_versions.md | 2 +- docs/tables/aws_glue_registry_schemas.md | 2 +- resources/services/glue/registries.go | 10 ++++++---- resources/services/glue/registries.hcl | 6 ++++-- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/docs/tables/aws_glue_registry_schema_versions.md b/docs/tables/aws_glue_registry_schema_versions.md index a48048dae..9519267e0 100644 --- a/docs/tables/aws_glue_registry_schema_versions.md +++ b/docs/tables/aws_glue_registry_schema_versions.md @@ -1,6 +1,6 @@ # Table: aws_glue_registry_schema_versions - +An object containing the details about a schema version ## Columns | Name | Type | Description | | ------------- | ------------- | ----- | diff --git a/docs/tables/aws_glue_registry_schemas.md b/docs/tables/aws_glue_registry_schemas.md index c3465eb36..21cbda540 100644 --- a/docs/tables/aws_glue_registry_schemas.md +++ b/docs/tables/aws_glue_registry_schemas.md @@ -1,6 +1,6 @@ # Table: aws_glue_registry_schemas - +An object that contains minimal details for a schema ## Columns | Name | Type | Description | | ------------- | ------------- | ----- | diff --git a/resources/services/glue/registries.go b/resources/services/glue/registries.go index 64c658e40..47da5ed31 100644 --- a/resources/services/glue/registries.go +++ b/resources/services/glue/registries.go @@ -74,8 +74,9 @@ func Registries() *schema.Table { }, Relations: []*schema.Table{ { - Name: "aws_glue_registry_schemas", - Resolver: fetchGlueRegistrySchemas, + Name: "aws_glue_registry_schemas", + Description: "An object that contains minimal details for a schema", + Resolver: fetchGlueRegistrySchemas, Columns: []schema.Column{ { Name: "registry_cq_id", @@ -158,8 +159,9 @@ func Registries() *schema.Table { }, Relations: []*schema.Table{ { - Name: "aws_glue_registry_schema_versions", - Resolver: fetchGlueRegistrySchemaVersions, + Name: "aws_glue_registry_schema_versions", + Description: "An object containing the details about a schema version", + Resolver: fetchGlueRegistrySchemaVersions, Columns: []schema.Column{ { Name: "registry_schema_cq_id", diff --git a/resources/services/glue/registries.hcl b/resources/services/glue/registries.hcl index 40db70c1c..9171cd6ca 100644 --- a/resources/services/glue/registries.hcl +++ b/resources/services/glue/registries.hcl @@ -41,7 +41,8 @@ resource "aws" "glue" "registries" { generate_resolver = true } user_relation "aws" "glue" "schemas" { - path = "github.com/aws/aws-sdk-go-v2/service/glue.GetSchemaOutput" + path = "github.com/aws/aws-sdk-go-v2/service/glue.GetSchemaOutput" + description = "An object that contains minimal details for a schema" userDefinedColumn "tags" { type = "json" description = "Resource tags." @@ -52,7 +53,8 @@ resource "aws" "glue" "registries" { } user_relation "aws" "glue" "versions" { - path = "github.com/aws/aws-sdk-go-v2/service/glue.GetSchemaVersionOutput" + path = "github.com/aws/aws-sdk-go-v2/service/glue.GetSchemaVersionOutput" + description = "An object containing the details about a schema version" userDefinedColumn "metadata" { type = "json" generate_resolver = true