From 63ed6f391dffdc1849d74918a99a46cdcb7ac9ba Mon Sep 17 00:00:00 2001 From: amanenk Date: Mon, 1 Aug 2022 18:09:22 +0300 Subject: [PATCH 1/3] feat: Added glue dev endpoints --- client/mocks/glue.go | 20 ++ client/services.go | 1 + docs/tables/aws_glue_dev_endpoints.md | 35 +++ resources/provider/provider.go | 1 + resources/services/glue/dev_endpoints.go | 229 ++++++++++++++++++ resources/services/glue/dev_endpoints.hcl | 55 +++++ resources/services/glue/dev_endpoints_test.go | 40 +++ 7 files changed, 381 insertions(+) create mode 100644 docs/tables/aws_glue_dev_endpoints.md create mode 100644 resources/services/glue/dev_endpoints.go create mode 100644 resources/services/glue/dev_endpoints.hcl create mode 100644 resources/services/glue/dev_endpoints_test.go diff --git a/client/mocks/glue.go b/client/mocks/glue.go index 3c3fe3a70..61f63b108 100644 --- a/client/mocks/glue.go +++ b/client/mocks/glue.go @@ -35,6 +35,26 @@ func (m *MockGlueClient) EXPECT() *MockGlueClientMockRecorder { return m.recorder } +// GetDevEndpoints mocks base method. +func (m *MockGlueClient) GetDevEndpoints(arg0 context.Context, arg1 *glue.GetDevEndpointsInput, arg2 ...func(*glue.Options)) (*glue.GetDevEndpointsOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetDevEndpoints", varargs...) + ret0, _ := ret[0].(*glue.GetDevEndpointsOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetDevEndpoints indicates an expected call of GetDevEndpoints. +func (mr *MockGlueClientMockRecorder) GetDevEndpoints(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDevEndpoints", reflect.TypeOf((*MockGlueClient)(nil).GetDevEndpoints), varargs...) +} + // GetTags mocks base method. func (m *MockGlueClient) GetTags(arg0 context.Context, arg1 *glue.GetTagsInput, arg2 ...func(*glue.Options)) (*glue.GetTagsOutput, error) { m.ctrl.T.Helper() diff --git a/client/services.go b/client/services.go index 69b5faca0..c8f9ff4c5 100644 --- a/client/services.go +++ b/client/services.go @@ -752,6 +752,7 @@ type GlueClient interface { GetTags(ctx context.Context, params *glue.GetTagsInput, optFns ...func(*glue.Options)) (*glue.GetTagsOutput, error) GetWorkflow(ctx context.Context, params *glue.GetWorkflowInput, optFns ...func(*glue.Options)) (*glue.GetWorkflowOutput, error) ListWorkflows(ctx context.Context, params *glue.ListWorkflowsInput, optFns ...func(*glue.Options)) (*glue.ListWorkflowsOutput, error) + GetDevEndpoints(ctx context.Context, params *glue.GetDevEndpointsInput, optFns ...func(*glue.Options)) (*glue.GetDevEndpointsOutput, error) } //go:generate mockgen -package=mocks -destination=./mocks/kinesis.go . KinesisClient diff --git a/docs/tables/aws_glue_dev_endpoints.md b/docs/tables/aws_glue_dev_endpoints.md new file mode 100644 index 000000000..200aef00a --- /dev/null +++ b/docs/tables/aws_glue_dev_endpoints.md @@ -0,0 +1,35 @@ + +# Table: aws_glue_dev_endpoints +A development endpoint where a developer can remotely debug extract, transform, and load (ETL) scripts +## Columns +| Name | Type | Description | +| ------------- | ------------- | ----- | +|account_id|text|The AWS Account ID of the resource.| +|region|text|The AWS Region of the resource.| +|arn|text|The Amazon Resource Name (ARN) of the workflow.| +|tags|jsonb|Resource tags.| +|arguments|jsonb|A map of arguments used to configure the DevEndpoint| +|availability_zone|text|The AWS Availability Zone where this DevEndpoint is located| +|created_timestamp|timestamp without time zone|The point in time at which this DevEndpoint was created| +|name|text|The name of the DevEndpoint| +|extra_jars_s3_path|text|The path to one or more Java jar files in an S3 bucket that should be loaded in your DevEndpoint| +|extra_python_libs_s3_path|text|The paths to one or more Python libraries in an Amazon S3 bucket that should be loaded in your DevEndpoint| +|failure_reason|text|The reason for a current failure in this DevEndpoint| +|glue_version|text|Glue version determines the versions of Apache Spark and Python that Glue supports| +|last_modified_timestamp|timestamp without time zone|The point in time at which this DevEndpoint was last modified| +|last_update_status|text|The status of the last update| +|number_of_nodes|bigint|The number of Glue Data Processing Units (DPUs) allocated to this DevEndpoint| +|number_of_workers|bigint|The number of workers of a defined workerType that are allocated to the development endpoint| +|private_address|text|A private IP address to access the DevEndpoint within a VPC if the DevEndpoint is created within one| +|public_address|text|The public IP address used by this DevEndpoint| +|public_key|text|The public key to be used by this DevEndpoint for authentication| +|public_keys|text[]|A list of public keys to be used by the DevEndpoints for authentication| +|role_arn|text|The Amazon Resource Name (ARN) of the IAM role used in this DevEndpoint| +|security_configuration|text|The name of the SecurityConfiguration structure to be used with this DevEndpoint| +|security_group_ids|text[]|A list of security group identifiers used in this DevEndpoint| +|status|text|The current status of this DevEndpoint| +|subnet_id|text|The subnet ID for this DevEndpoint| +|vpc_id|text|The ID of the virtual private cloud (VPC) used by this DevEndpoint| +|worker_type|text|The type of predefined worker that is allocated to the development endpoint Accepts a value of Standard, G1X, or G2X * For the Standard worker type, each worker provides 4 vCPU, 16 GB of memory and a 50GB disk, and 2 executors per worker * For the G1X worker type, each worker maps to 1 DPU (4 vCPU, 16 GB of memory, 64 GB disk), and provides 1 executor per worker| +|yarn_endpoint_address|text|The YARN endpoint address used by this DevEndpoint| +|zeppelin_remote_spark_interpreter_port|bigint|The Apache Zeppelin port for the remote Apache Spark interpreter| diff --git a/resources/provider/provider.go b/resources/provider/provider.go index 898ef6f66..b5bd3f298 100644 --- a/resources/provider/provider.go +++ b/resources/provider/provider.go @@ -168,6 +168,7 @@ func Provider() *provider.Provider { "emr.block_public_access_configs": emr.EmrBlockPublicAccessConfigs(), "emr.clusters": emr.EmrClusters(), "fsx.backups": fsx.FsxBackups(), + "glue.dev_endpoints": glue.DevEndpoints(), "glue.workflows": glue.Workflows(), "guardduty.detectors": guardduty.GuarddutyDetectors(), "iam.accounts": iam.IamAccounts(), diff --git a/resources/services/glue/dev_endpoints.go b/resources/services/glue/dev_endpoints.go new file mode 100644 index 000000000..1c4d9164c --- /dev/null +++ b/resources/services/glue/dev_endpoints.go @@ -0,0 +1,229 @@ +package glue + +import ( + "context" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/glue" + "github.com/aws/aws-sdk-go-v2/service/glue/types" + "github.com/cloudquery/cq-provider-aws/client" + "github.com/cloudquery/cq-provider-sdk/provider/diag" + "github.com/cloudquery/cq-provider-sdk/provider/schema" +) + +//go:generate cq-gen --resource dev_endpoints --config dev_endpoints.hcl --output . +func DevEndpoints() *schema.Table { + return &schema.Table{ + Name: "aws_glue_dev_endpoints", + Description: "A development endpoint where a developer can remotely debug extract, transform, and load (ETL) scripts", + Resolver: fetchGlueDevEndpoints, + Multiplex: client.ServiceAccountRegionMultiplexer("glue"), + IgnoreError: client.IgnoreAccessDeniedServiceDisabled, + DeleteFilter: client.DeleteAccountRegionFilter, + Options: schema.TableCreationOptions{PrimaryKeys: []string{"arn"}}, + Columns: []schema.Column{ + { + Name: "account_id", + Description: "The AWS Account ID of the resource.", + Type: schema.TypeString, + Resolver: client.ResolveAWSAccount, + }, + { + Name: "region", + Description: "The AWS Region of the resource.", + Type: schema.TypeString, + Resolver: client.ResolveAWSRegion, + }, + { + Name: "arn", + Description: "The Amazon Resource Name (ARN) of the workflow.", + Type: schema.TypeString, + Resolver: resolveGlueDevEndpointArn, + }, + { + Name: "tags", + Description: "Resource tags.", + Type: schema.TypeJSON, + Resolver: resolveGlueDevEndpointTags, + }, + { + Name: "arguments", + Description: "A map of arguments used to configure the DevEndpoint", + Type: schema.TypeJSON, + }, + { + Name: "availability_zone", + Description: "The AWS Availability Zone where this DevEndpoint is located", + Type: schema.TypeString, + }, + { + Name: "created_timestamp", + Description: "The point in time at which this DevEndpoint was created", + Type: schema.TypeTimestamp, + }, + { + Name: "name", + Description: "The name of the DevEndpoint", + Type: schema.TypeString, + Resolver: schema.PathResolver("EndpointName"), + }, + { + Name: "extra_jars_s3_path", + Description: "The path to one or more Java jar files in an S3 bucket that should be loaded in your DevEndpoint", + Type: schema.TypeString, + }, + { + Name: "extra_python_libs_s3_path", + Description: "The paths to one or more Python libraries in an Amazon S3 bucket that should be loaded in your DevEndpoint", + Type: schema.TypeString, + }, + { + Name: "failure_reason", + Description: "The reason for a current failure in this DevEndpoint", + Type: schema.TypeString, + }, + { + Name: "glue_version", + Description: "Glue version determines the versions of Apache Spark and Python that Glue supports", + Type: schema.TypeString, + }, + { + Name: "last_modified_timestamp", + Description: "The point in time at which this DevEndpoint was last modified", + Type: schema.TypeTimestamp, + }, + { + Name: "last_update_status", + Description: "The status of the last update", + Type: schema.TypeString, + }, + { + Name: "number_of_nodes", + Description: "The number of Glue Data Processing Units (DPUs) allocated to this DevEndpoint", + Type: schema.TypeBigInt, + }, + { + Name: "number_of_workers", + Description: "The number of workers of a defined workerType that are allocated to the development endpoint", + Type: schema.TypeBigInt, + }, + { + Name: "private_address", + Description: "A private IP address to access the DevEndpoint within a VPC if the DevEndpoint is created within one", + Type: schema.TypeString, + }, + { + Name: "public_address", + Description: "The public IP address used by this DevEndpoint", + Type: schema.TypeString, + }, + { + Name: "public_key", + Description: "The public key to be used by this DevEndpoint for authentication", + Type: schema.TypeString, + }, + { + Name: "public_keys", + Description: "A list of public keys to be used by the DevEndpoints for authentication", + Type: schema.TypeStringArray, + }, + { + Name: "role_arn", + Description: "The Amazon Resource Name (ARN) of the IAM role used in this DevEndpoint", + Type: schema.TypeString, + }, + { + Name: "security_configuration", + Description: "The name of the SecurityConfiguration structure to be used with this DevEndpoint", + Type: schema.TypeString, + }, + { + Name: "security_group_ids", + Description: "A list of security group identifiers used in this DevEndpoint", + Type: schema.TypeStringArray, + }, + { + Name: "status", + Description: "The current status of this DevEndpoint", + Type: schema.TypeString, + }, + { + Name: "subnet_id", + Description: "The subnet ID for this DevEndpoint", + Type: schema.TypeString, + }, + { + Name: "vpc_id", + Description: "The ID of the virtual private cloud (VPC) used by this DevEndpoint", + Type: schema.TypeString, + }, + { + Name: "worker_type", + Description: "The type of predefined worker that is allocated to the development endpoint Accepts a value of Standard, G1X, or G2X * For the Standard worker type, each worker provides 4 vCPU, 16 GB of memory and a 50GB disk, and 2 executors per worker * For the G1X worker type, each worker maps to 1 DPU (4 vCPU, 16 GB of memory, 64 GB disk), and provides 1 executor per worker", + Type: schema.TypeString, + }, + { + Name: "yarn_endpoint_address", + Description: "The YARN endpoint address used by this DevEndpoint", + Type: schema.TypeString, + }, + { + Name: "zeppelin_remote_spark_interpreter_port", + Description: "The Apache Zeppelin port for the remote Apache Spark interpreter", + Type: schema.TypeBigInt, + }, + }, + } +} + +// ==================================================================================================================== +// Table Resolver Functions +// ==================================================================================================================== + +func fetchGlueDevEndpoints(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- interface{}) error { + cl := meta.(*client.Client) + svc := cl.Services().Glue + input := glue.GetDevEndpointsInput{} + for { + result, err := svc.GetDevEndpoints(ctx, &input) + if err != nil { + if cl.IsNotFoundError(err) { + return nil + } + return diag.WrapError(err) + } + res <- result.DevEndpoints + if aws.ToString(result.NextToken) == "" { + break + } + input.NextToken = result.NextToken + } + return nil +} +func resolveGlueDevEndpointArn(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { + cl := meta.(*client.Client) + arn := aws.String(devEndpointARN(cl, aws.ToString(resource.Item.(types.DevEndpoint).EndpointName))) + return diag.WrapError(resource.Set(c.Name, arn)) +} +func resolveGlueDevEndpointTags(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { + cl := meta.(*client.Client) + svc := cl.Services().Glue + result, err := svc.GetTags(ctx, &glue.GetTagsInput{ + ResourceArn: aws.String(devEndpointARN(cl, aws.ToString(resource.Item.(types.DevEndpoint).EndpointName))), + }) + if err != nil { + if cl.IsNotFoundError(err) { + return nil + } + return diag.WrapError(err) + } + return diag.WrapError(resource.Set(c.Name, result.Tags)) +} + +// ==================================================================================================================== +// User Defined Helpers +// ==================================================================================================================== + +func devEndpointARN(cl *client.Client, name string) string { + return cl.ARN(client.GlueService, "devEndpoint", name) +} diff --git a/resources/services/glue/dev_endpoints.hcl b/resources/services/glue/dev_endpoints.hcl new file mode 100644 index 000000000..681529749 --- /dev/null +++ b/resources/services/glue/dev_endpoints.hcl @@ -0,0 +1,55 @@ +service = "aws" +output_directory = "." +add_generate = true + + +description_modifier "remove_read_only" { + words = [" This member is required."] +} + +resource "aws" "glue" "dev_endpoints" { + path = "github.com/aws/aws-sdk-go-v2/service/glue/types.DevEndpoint" + ignoreError "IgnoreAccessDenied" { + path = "github.com/cloudquery/cq-provider-aws/client.IgnoreAccessDeniedServiceDisabled" + } + deleteFilter "AccountRegionFilter" { + path = "github.com/cloudquery/cq-provider-aws/client.DeleteAccountRegionFilter" + } + multiplex "AwsAccountRegion" { + path = "github.com/cloudquery/cq-provider-aws/client.ServiceAccountRegionMultiplexer" + params = ["glue"] + } + options { + primary_keys = ["arn"] + } + userDefinedColumn "account_id" { + description = "The AWS Account ID of the resource." + type = "string" + resolver "resolveAWSAccount" { + path = "github.com/cloudquery/cq-provider-aws/client.ResolveAWSAccount" + } + } + userDefinedColumn "region" { + type = "string" + description = "The AWS Region of the resource." + resolver "resolveAWSRegion" { + path = "github.com/cloudquery/cq-provider-aws/client.ResolveAWSRegion" + } + } + userDefinedColumn "arn" { + type = "string" + description = "The Amazon Resource Name (ARN) of the workflow." + generate_resolver = true + } + + column "endpoint_name" { + rename = "name" + } + + + userDefinedColumn "tags" { + type = "json" + description = "Resource tags." + generate_resolver = true + } +} diff --git a/resources/services/glue/dev_endpoints_test.go b/resources/services/glue/dev_endpoints_test.go new file mode 100644 index 000000000..491b0acad --- /dev/null +++ b/resources/services/glue/dev_endpoints_test.go @@ -0,0 +1,40 @@ +package glue + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/service/glue" + "github.com/cloudquery/cq-provider-aws/client" + "github.com/cloudquery/cq-provider-aws/client/mocks" + "github.com/cloudquery/faker/v3" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" +) + +func buildDevEndpointsMock(t *testing.T, ctrl *gomock.Controller) client.Services { + m := mocks.NewMockGlueClient(ctrl) + + var devEndpoint glue.GetDevEndpointsOutput + require.NoError(t, faker.FakeData(&devEndpoint)) + devEndpoint.NextToken = nil + m.EXPECT().GetDevEndpoints( + gomock.Any(), + &glue.GetDevEndpointsInput{}, + ).Return(&devEndpoint, nil) + + m.EXPECT().GetTags( + gomock.Any(), + gomock.Any(), + ).Return( + &glue.GetTagsOutput{Tags: map[string]string{"key": "value"}}, + nil, + ) + + return client.Services{ + Glue: m, + } +} + +func TestDevEndpoints(t *testing.T) { + client.AwsMockTestHelper(t, DevEndpoints(), buildDevEndpointsMock, client.TestOptions{}) +} From 8c3bdd62760ff293b8e1b445f85dfda0b4db3b35 Mon Sep 17 00:00:00 2001 From: amanenk Date: Wed, 3 Aug 2022 14:30:20 +0300 Subject: [PATCH 2/3] fix description --- docs/tables/aws_glue_dev_endpoints.md | 2 +- resources/services/glue/dev_endpoints.go | 2 +- resources/services/glue/dev_endpoints.hcl | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/tables/aws_glue_dev_endpoints.md b/docs/tables/aws_glue_dev_endpoints.md index 200aef00a..5d16b097c 100644 --- a/docs/tables/aws_glue_dev_endpoints.md +++ b/docs/tables/aws_glue_dev_endpoints.md @@ -30,6 +30,6 @@ A development endpoint where a developer can remotely debug extract, transform, |status|text|The current status of this DevEndpoint| |subnet_id|text|The subnet ID for this DevEndpoint| |vpc_id|text|The ID of the virtual private cloud (VPC) used by this DevEndpoint| -|worker_type|text|The type of predefined worker that is allocated to the development endpoint Accepts a value of Standard, G1X, or G2X * For the Standard worker type, each worker provides 4 vCPU, 16 GB of memory and a 50GB disk, and 2 executors per worker * For the G1X worker type, each worker maps to 1 DPU (4 vCPU, 16 GB of memory, 64 GB disk), and provides 1 executor per worker| +|worker_type|text|The type of predefined worker that is allocated to the development endpoint Accepts a value of Standard, G1X, or G2X| |yarn_endpoint_address|text|The YARN endpoint address used by this DevEndpoint| |zeppelin_remote_spark_interpreter_port|bigint|The Apache Zeppelin port for the remote Apache Spark interpreter| diff --git a/resources/services/glue/dev_endpoints.go b/resources/services/glue/dev_endpoints.go index 1c4d9164c..683afefce 100644 --- a/resources/services/glue/dev_endpoints.go +++ b/resources/services/glue/dev_endpoints.go @@ -159,7 +159,7 @@ func DevEndpoints() *schema.Table { }, { Name: "worker_type", - Description: "The type of predefined worker that is allocated to the development endpoint Accepts a value of Standard, G1X, or G2X * For the Standard worker type, each worker provides 4 vCPU, 16 GB of memory and a 50GB disk, and 2 executors per worker * For the G1X worker type, each worker maps to 1 DPU (4 vCPU, 16 GB of memory, 64 GB disk), and provides 1 executor per worker", + Description: "The type of predefined worker that is allocated to the development endpoint Accepts a value of Standard, G1X, or G2X", Type: schema.TypeString, }, { diff --git a/resources/services/glue/dev_endpoints.hcl b/resources/services/glue/dev_endpoints.hcl index 681529749..8cbb01bbe 100644 --- a/resources/services/glue/dev_endpoints.hcl +++ b/resources/services/glue/dev_endpoints.hcl @@ -46,6 +46,9 @@ resource "aws" "glue" "dev_endpoints" { rename = "name" } + column "worker_type" { + description = "The type of predefined worker that is allocated to the development endpoint Accepts a value of Standard, G1X, or G2X" + } userDefinedColumn "tags" { type = "json" From c10f5a42415b66c487f8bf2facc6946744f5a68e Mon Sep 17 00:00:00 2001 From: amanenk Date: Wed, 3 Aug 2022 14:32:22 +0300 Subject: [PATCH 3/3] mock generation --- client/mocks/glue.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/client/mocks/glue.go b/client/mocks/glue.go index a6691cba9..b2bc6ec9a 100644 --- a/client/mocks/glue.go +++ b/client/mocks/glue.go @@ -75,6 +75,26 @@ func (mr *MockGlueClientMockRecorder) GetDatabases(arg0, arg1 interface{}, arg2 return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDatabases", reflect.TypeOf((*MockGlueClient)(nil).GetDatabases), varargs...) } +// GetDevEndpoints mocks base method. +func (m *MockGlueClient) GetDevEndpoints(arg0 context.Context, arg1 *glue.GetDevEndpointsInput, arg2 ...func(*glue.Options)) (*glue.GetDevEndpointsOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetDevEndpoints", varargs...) + ret0, _ := ret[0].(*glue.GetDevEndpointsOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetDevEndpoints indicates an expected call of GetDevEndpoints. +func (mr *MockGlueClientMockRecorder) GetDevEndpoints(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDevEndpoints", reflect.TypeOf((*MockGlueClient)(nil).GetDevEndpoints), varargs...) +} + // GetJobRuns mocks base method. func (m *MockGlueClient) GetJobRuns(arg0 context.Context, arg1 *glue.GetJobRunsInput, arg2 ...func(*glue.Options)) (*glue.GetJobRunsOutput, error) { m.ctrl.T.Helper()