From 8e1391a8690bb1246e017573885b1f9392f181d6 Mon Sep 17 00:00:00 2001 From: James Doldissen Date: Mon, 2 Sep 2024 23:10:14 +0200 Subject: [PATCH 01/10] feat: Add SDK for External Volumes --- pkg/sdk/client.go | 2 + pkg/sdk/external_volumes_def.go | 159 +++++ pkg/sdk/external_volumes_dto_builders_gen.go | 197 ++++++ pkg/sdk/external_volumes_dto_gen.go | 83 +++ pkg/sdk/external_volumes_gen.go | 118 ++++ pkg/sdk/external_volumes_gen_test.go | 391 ++++++++++++ pkg/sdk/external_volumes_impl_gen.go | 174 ++++++ pkg/sdk/external_volumes_validations_gen.go | 89 +++ pkg/sdk/poc/main.go | 1 + .../external_volumes_gen_integration_test.go | 566 ++++++++++++++++++ 10 files changed, 1780 insertions(+) create mode 100644 pkg/sdk/external_volumes_def.go create mode 100644 pkg/sdk/external_volumes_dto_builders_gen.go create mode 100644 pkg/sdk/external_volumes_dto_gen.go create mode 100644 pkg/sdk/external_volumes_gen.go create mode 100644 pkg/sdk/external_volumes_gen_test.go create mode 100644 pkg/sdk/external_volumes_impl_gen.go create mode 100644 pkg/sdk/external_volumes_validations_gen.go create mode 100644 pkg/sdk/testint/external_volumes_gen_integration_test.go diff --git a/pkg/sdk/client.go b/pkg/sdk/client.go index c541793ef5..9687ae64f0 100644 --- a/pkg/sdk/client.go +++ b/pkg/sdk/client.go @@ -52,6 +52,7 @@ type Client struct { DataMetricFunctionReferences DataMetricFunctionReferences DynamicTables DynamicTables ExternalFunctions ExternalFunctions + ExternalVolumes ExternalVolumes ExternalTables ExternalTables EventTables EventTables FailoverGroups FailoverGroups @@ -209,6 +210,7 @@ func (c *Client) initialize() { c.DataMetricFunctionReferences = &dataMetricFunctionReferences{client: c} c.DynamicTables = &dynamicTables{client: c} c.ExternalFunctions = &externalFunctions{client: c} + c.ExternalVolumes = &externalVolumes{client: c} c.ExternalTables = &externalTables{client: c} c.EventTables = &eventTables{client: c} c.FailoverGroups = &failoverGroups{client: c} diff --git a/pkg/sdk/external_volumes_def.go b/pkg/sdk/external_volumes_def.go new file mode 100644 index 0000000000..5c43d62361 --- /dev/null +++ b/pkg/sdk/external_volumes_def.go @@ -0,0 +1,159 @@ +package sdk + +import g "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/sdk/poc/generator" + +//go:generate go run ./poc/main.go + +type S3EncryptionType string +type S3StorageProvider string +type GCSEncryptionType string + +var ( + S3EncryptionTypeSseS3 S3EncryptionType = "AWS_SSE_S3" + S3EncryptionTypeSseKms S3EncryptionType = "AWS_SSE_KMS" + S3EncryptionNone S3EncryptionType = "NONE" + GCSEncryptionTypeSseKms GCSEncryptionType = "GCS_SSE_KMS" + GCSEncryptionTypeNone GCSEncryptionType = "NONE" + S3StorageProviderS3 S3StorageProvider = "S3" + S3StorageProviderS3GOV S3StorageProvider = "S3GOV" +) + +var externalS3StorageLocationDef = g.NewQueryStruct("S3StorageLocationParams"). + TextAssignment("NAME", g.ParameterOptions().SingleQuotes().Required()). + Assignment("STORAGE_PROVIDER", g.KindOfT[S3StorageProvider](), g.ParameterOptions().SingleQuotes().Required()). + TextAssignment("STORAGE_AWS_ROLE_ARN", g.ParameterOptions().SingleQuotes().Required()). + TextAssignment("STORAGE_BASE_URL", g.ParameterOptions().SingleQuotes().Required()). + OptionalTextAssignment("STORAGE_AWS_EXTERNAL_ID", g.ParameterOptions().SingleQuotes()). + QueryStructField( + "Encryption", + g.NewQueryStruct("ExternalVolumeS3Encryption"). + Assignment("TYPE", g.KindOfT[S3EncryptionType](), g.ParameterOptions().SingleQuotes().Required()). + OptionalTextAssignment("KMS_KEY_ID", g.ParameterOptions().SingleQuotes()), + g.ListOptions().Parentheses().NoComma().SQL("ENCRYPTION =").Required(), + ) + +var externalGCSStorageLocationDef = g.NewQueryStruct("GCSStorageLocationParams"). + TextAssignment("NAME", g.ParameterOptions().SingleQuotes().Required()). + PredefinedQueryStructField("StorageProviderGcs", "string", g.StaticOptions().SQL("STORAGE_PROVIDER = 'GCS'")). + TextAssignment("STORAGE_BASE_URL", g.ParameterOptions().SingleQuotes().Required()). + QueryStructField( + "Encryption", + g.NewQueryStruct("ExternalVolumeGCSEncryption"). + Assignment("TYPE", g.KindOfT[GCSEncryptionType](), g.ParameterOptions().SingleQuotes().Required()). + OptionalTextAssignment("KMS_KEY_ID", g.ParameterOptions().SingleQuotes()), + g.ListOptions().Parentheses().NoComma().SQL("ENCRYPTION =").Required(), + ) + +var externalAzureStorageLocationDef = g.NewQueryStruct("AzureStorageLocationParams"). + TextAssignment("NAME", g.ParameterOptions().SingleQuotes().Required()). + PredefinedQueryStructField("StorageProviderAzure", "string", g.StaticOptions().SQL("STORAGE_PROVIDER = 'AZURE'")). + TextAssignment("AZURE_TENANT_ID", g.ParameterOptions().SingleQuotes().Required()). + TextAssignment("STORAGE_BASE_URL", g.ParameterOptions().SingleQuotes().Required()) + +var storageLocationDef = g.NewQueryStruct("ExternalVolumeStorageLocation"). // Can't name StorageLocation due to naming clash with type in storage integration + OptionalQueryStructField( + "S3StorageLocationParams", + externalS3StorageLocationDef, + g.ListOptions().Parentheses().NoComma(), + ). + OptionalQueryStructField( + "GCSStorageLocationParams", + externalGCSStorageLocationDef, + g.ListOptions().Parentheses().NoComma(), + ). + OptionalQueryStructField( + "AzureStorageLocationParams", + externalAzureStorageLocationDef, + g.ListOptions().Parentheses().NoComma(), + ). + WithValidation(g.ExactlyOneValueSet, "S3StorageLocationParams", "GCSStorageLocationParams", "AzureStorageLocationParams") + +var ExternalVolumesDef = g.NewInterface( + "ExternalVolumes", + "ExternalVolume", + g.KindOfT[AccountObjectIdentifier](), +). + CreateOperation( + "https://docs.snowflake.com/en/sql-reference/sql/create-external-volume", + g.NewQueryStruct("CreateExternalVolume"). + Create(). + OrReplace(). + SQL("EXTERNAL VOLUME"). + IfNotExists(). + Name(). + ListAssignment("STORAGE_LOCATIONS", "ExternalVolumeStorageLocation", g.ParameterOptions().Parentheses().Required()). + OptionalBooleanAssignment("ALLOW_WRITES", nil). + OptionalComment(). + WithValidation(g.ConflictingFields, "OrReplace", "IfNotExists"). + WithValidation(g.ValidIdentifier, "name"), + storageLocationDef, + ). + AlterOperation( + "https://docs.snowflake.com/en/sql-reference/sql/alter-external-volume", + g.NewQueryStruct("AlterExternalVolume"). + Alter(). + SQL("EXTERNAL VOLUME"). + IfExists(). + Name(). + OptionalTextAssignment("REMOVE STORAGE_LOCATION", g.ParameterOptions().SingleQuotes().NoEquals()). + OptionalQueryStructField( + "Set", + g.NewQueryStruct("AlterExternalVolumeSet"). + OptionalBooleanAssignment("ALLOW_WRITES", g.ParameterOptions()). + OptionalComment(), + g.KeywordOptions().SQL("SET"), + ). + OptionalQueryStructField( + "AddStorageLocation", + storageLocationDef, + g.ParameterOptions().SQL("ADD STORAGE_LOCATION"), + ). + WithValidation(g.ExactlyOneValueSet, "RemoveStorageLocation", "Set", "AddStorageLocation"). + WithValidation(g.ValidIdentifier, "name"), + ). + DropOperation( + "https://docs.snowflake.com/en/sql-reference/sql/drop-external-volume", + g.NewQueryStruct("DropExternalVolume"). + Drop(). + SQL("EXTERNAL VOLUME"). + IfExists(). + Name(). + WithValidation(g.ValidIdentifier, "name"), + ). + DescribeOperation( + g.DescriptionMappingKindSlice, + "https://docs.snowflake.com/en/sql-reference/sql/desc-external-volume", + g.DbStruct("externalVolumeDescRow"). + Field("parent_property", "string"). + Field("property", "string"). + Field("property_type", "string"). + Field("property_value", "string"). + Field("property_default", "string"), + g.PlainStruct("ExternalVolumeProperty"). + Field("Parent", "string"). + Field("Name", "string"). + Field("Type", "string"). + Field("Value", "string"). + Field("Default", "string"), + g.NewQueryStruct("DescExternalVolume"). + Describe(). + SQL("EXTERNAL VOLUME"). + Name(). + WithValidation(g.ValidIdentifier, "name"), + ). + ShowOperation( + "https://docs.snowflake.com/en/sql-reference/sql/show-external-volumes", + g.DbStruct("externalVolumeShowRow"). + Field("name", "string"). + Field("allow_writes", "string"). + Field("comment", "string"), + g.PlainStruct("ExternalVolume"). + Field("Name", "string"). + Field("AllowWrites", "string"). + Field("Comment", "string"), + g.NewQueryStruct("ShowExternalVolumes"). + Show(). + SQL("EXTERNAL VOLUMES"). + OptionalLike(), + ). + ShowByIdOperation() diff --git a/pkg/sdk/external_volumes_dto_builders_gen.go b/pkg/sdk/external_volumes_dto_builders_gen.go new file mode 100644 index 0000000000..abb1ba98d1 --- /dev/null +++ b/pkg/sdk/external_volumes_dto_builders_gen.go @@ -0,0 +1,197 @@ +// Code generated by dto builder generator; DO NOT EDIT. + +package sdk + +import () + +func NewCreateExternalVolumeRequest( + name AccountObjectIdentifier, + StorageLocations []ExternalVolumeStorageLocation, +) *CreateExternalVolumeRequest { + s := CreateExternalVolumeRequest{} + s.name = name + s.StorageLocations = StorageLocations + return &s +} + +func (s *CreateExternalVolumeRequest) WithOrReplace(OrReplace bool) *CreateExternalVolumeRequest { + s.OrReplace = &OrReplace + return s +} + +func (s *CreateExternalVolumeRequest) WithIfNotExists(IfNotExists bool) *CreateExternalVolumeRequest { + s.IfNotExists = &IfNotExists + return s +} + +func (s *CreateExternalVolumeRequest) WithAllowWrites(AllowWrites bool) *CreateExternalVolumeRequest { + s.AllowWrites = &AllowWrites + return s +} + +func (s *CreateExternalVolumeRequest) WithComment(Comment string) *CreateExternalVolumeRequest { + s.Comment = &Comment + return s +} + +func NewAlterExternalVolumeRequest( + name AccountObjectIdentifier, +) *AlterExternalVolumeRequest { + s := AlterExternalVolumeRequest{} + s.name = name + return &s +} + +func (s *AlterExternalVolumeRequest) WithIfExists(IfExists bool) *AlterExternalVolumeRequest { + s.IfExists = &IfExists + return s +} + +func (s *AlterExternalVolumeRequest) WithRemoveStorageLocation(RemoveStorageLocation string) *AlterExternalVolumeRequest { + s.RemoveStorageLocation = &RemoveStorageLocation + return s +} + +func (s *AlterExternalVolumeRequest) WithSet(Set AlterExternalVolumeSetRequest) *AlterExternalVolumeRequest { + s.Set = &Set + return s +} + +func (s *AlterExternalVolumeRequest) WithAddStorageLocation(AddStorageLocation ExternalVolumeStorageLocationRequest) *AlterExternalVolumeRequest { + s.AddStorageLocation = &AddStorageLocation + return s +} + +func NewAlterExternalVolumeSetRequest() *AlterExternalVolumeSetRequest { + return &AlterExternalVolumeSetRequest{} +} + +func (s *AlterExternalVolumeSetRequest) WithAllowWrites(AllowWrites bool) *AlterExternalVolumeSetRequest { + s.AllowWrites = &AllowWrites + return s +} + +func (s *AlterExternalVolumeSetRequest) WithComment(Comment string) *AlterExternalVolumeSetRequest { + s.Comment = &Comment + return s +} + +func NewExternalVolumeStorageLocationRequest() *ExternalVolumeStorageLocationRequest { + return &ExternalVolumeStorageLocationRequest{} +} + +func (s *ExternalVolumeStorageLocationRequest) WithS3StorageLocationParams(S3StorageLocationParams S3StorageLocationParamsRequest) *ExternalVolumeStorageLocationRequest { + s.S3StorageLocationParams = &S3StorageLocationParams + return s +} + +func (s *ExternalVolumeStorageLocationRequest) WithGCSStorageLocationParams(GCSStorageLocationParams GCSStorageLocationParamsRequest) *ExternalVolumeStorageLocationRequest { + s.GCSStorageLocationParams = &GCSStorageLocationParams + return s +} + +func (s *ExternalVolumeStorageLocationRequest) WithAzureStorageLocationParams(AzureStorageLocationParams AzureStorageLocationParamsRequest) *ExternalVolumeStorageLocationRequest { + s.AzureStorageLocationParams = &AzureStorageLocationParams + return s +} + +func NewS3StorageLocationParamsRequest( + Name string, + StorageProvider S3StorageProvider, + StorageAwsRoleArn string, + StorageBaseUrl string, + Encryption ExternalVolumeS3EncryptionRequest, +) *S3StorageLocationParamsRequest { + s := S3StorageLocationParamsRequest{} + s.Name = Name + s.StorageProvider = StorageProvider + s.StorageAwsRoleArn = StorageAwsRoleArn + s.StorageBaseUrl = StorageBaseUrl + s.Encryption = Encryption + return &s +} + +func (s *S3StorageLocationParamsRequest) WithStorageAwsExternalId(StorageAwsExternalId string) *S3StorageLocationParamsRequest { + s.StorageAwsExternalId = &StorageAwsExternalId + return s +} + +func NewExternalVolumeS3EncryptionRequest( + Type S3EncryptionType, +) *ExternalVolumeS3EncryptionRequest { + s := ExternalVolumeS3EncryptionRequest{} + s.Type = Type + return &s +} + +func (s *ExternalVolumeS3EncryptionRequest) WithKmsKeyId(KmsKeyId string) *ExternalVolumeS3EncryptionRequest { + s.KmsKeyId = &KmsKeyId + return s +} + +func NewGCSStorageLocationParamsRequest( + Name string, + StorageBaseUrl string, + Encryption ExternalVolumeGCSEncryptionRequest, +) *GCSStorageLocationParamsRequest { + s := GCSStorageLocationParamsRequest{} + s.Name = Name + s.StorageBaseUrl = StorageBaseUrl + s.Encryption = Encryption + return &s +} + +func NewExternalVolumeGCSEncryptionRequest( + Type GCSEncryptionType, +) *ExternalVolumeGCSEncryptionRequest { + s := ExternalVolumeGCSEncryptionRequest{} + s.Type = Type + return &s +} + +func (s *ExternalVolumeGCSEncryptionRequest) WithKmsKeyId(KmsKeyId string) *ExternalVolumeGCSEncryptionRequest { + s.KmsKeyId = &KmsKeyId + return s +} + +func NewAzureStorageLocationParamsRequest( + Name string, + AzureTenantId string, + StorageBaseUrl string, +) *AzureStorageLocationParamsRequest { + s := AzureStorageLocationParamsRequest{} + s.Name = Name + s.AzureTenantId = AzureTenantId + s.StorageBaseUrl = StorageBaseUrl + return &s +} + +func NewDropExternalVolumeRequest( + name AccountObjectIdentifier, +) *DropExternalVolumeRequest { + s := DropExternalVolumeRequest{} + s.name = name + return &s +} + +func (s *DropExternalVolumeRequest) WithIfExists(IfExists bool) *DropExternalVolumeRequest { + s.IfExists = &IfExists + return s +} + +func NewDescribeExternalVolumeRequest( + name AccountObjectIdentifier, +) *DescribeExternalVolumeRequest { + s := DescribeExternalVolumeRequest{} + s.name = name + return &s +} + +func NewShowExternalVolumeRequest() *ShowExternalVolumeRequest { + return &ShowExternalVolumeRequest{} +} + +func (s *ShowExternalVolumeRequest) WithLike(Like Like) *ShowExternalVolumeRequest { + s.Like = &Like + return s +} diff --git a/pkg/sdk/external_volumes_dto_gen.go b/pkg/sdk/external_volumes_dto_gen.go new file mode 100644 index 0000000000..ec01cb5603 --- /dev/null +++ b/pkg/sdk/external_volumes_dto_gen.go @@ -0,0 +1,83 @@ +package sdk + +//go:generate go run ./dto-builder-generator/main.go + +var ( + _ optionsProvider[CreateExternalVolumeOptions] = new(CreateExternalVolumeRequest) + _ optionsProvider[AlterExternalVolumeOptions] = new(AlterExternalVolumeRequest) + _ optionsProvider[DropExternalVolumeOptions] = new(DropExternalVolumeRequest) + _ optionsProvider[DescribeExternalVolumeOptions] = new(DescribeExternalVolumeRequest) + _ optionsProvider[ShowExternalVolumeOptions] = new(ShowExternalVolumeRequest) +) + +type CreateExternalVolumeRequest struct { + OrReplace *bool + IfNotExists *bool + name AccountObjectIdentifier // required + StorageLocations []ExternalVolumeStorageLocation // required + AllowWrites *bool + Comment *string +} + +type AlterExternalVolumeRequest struct { + IfExists *bool + name AccountObjectIdentifier // required + RemoveStorageLocation *string + Set *AlterExternalVolumeSetRequest + AddStorageLocation *ExternalVolumeStorageLocationRequest +} + +type AlterExternalVolumeSetRequest struct { + AllowWrites *bool + Comment *string +} + +type ExternalVolumeStorageLocationRequest struct { + S3StorageLocationParams *S3StorageLocationParamsRequest + GCSStorageLocationParams *GCSStorageLocationParamsRequest + AzureStorageLocationParams *AzureStorageLocationParamsRequest +} + +type S3StorageLocationParamsRequest struct { + Name string // required + StorageProvider S3StorageProvider // required + StorageAwsRoleArn string // required + StorageBaseUrl string // required + StorageAwsExternalId *string + Encryption ExternalVolumeS3EncryptionRequest // required +} + +type ExternalVolumeS3EncryptionRequest struct { + Type S3EncryptionType // required + KmsKeyId *string +} + +type GCSStorageLocationParamsRequest struct { + Name string // required + StorageBaseUrl string // required + Encryption ExternalVolumeGCSEncryptionRequest // required +} + +type ExternalVolumeGCSEncryptionRequest struct { + Type GCSEncryptionType // required + KmsKeyId *string +} + +type AzureStorageLocationParamsRequest struct { + Name string // required + AzureTenantId string // required + StorageBaseUrl string // required +} + +type DropExternalVolumeRequest struct { + IfExists *bool + name AccountObjectIdentifier // required +} + +type DescribeExternalVolumeRequest struct { + name AccountObjectIdentifier // required +} + +type ShowExternalVolumeRequest struct { + Like *Like +} diff --git a/pkg/sdk/external_volumes_gen.go b/pkg/sdk/external_volumes_gen.go new file mode 100644 index 0000000000..659163da97 --- /dev/null +++ b/pkg/sdk/external_volumes_gen.go @@ -0,0 +1,118 @@ +package sdk + +import "context" + +type ExternalVolumes interface { + Create(ctx context.Context, request *CreateExternalVolumeRequest) error + Alter(ctx context.Context, request *AlterExternalVolumeRequest) error + Drop(ctx context.Context, request *DropExternalVolumeRequest) error + Describe(ctx context.Context, id AccountObjectIdentifier) ([]ExternalVolumeProperty, error) + Show(ctx context.Context, request *ShowExternalVolumeRequest) ([]ExternalVolume, error) + ShowByID(ctx context.Context, id AccountObjectIdentifier) (*ExternalVolume, error) +} + +// CreateExternalVolumeOptions is based on https://docs.snowflake.com/en/sql-reference/sql/create-external-volume. +type CreateExternalVolumeOptions struct { + create bool `ddl:"static" sql:"CREATE"` + OrReplace *bool `ddl:"keyword" sql:"OR REPLACE"` + externalVolume bool `ddl:"static" sql:"EXTERNAL VOLUME"` + IfNotExists *bool `ddl:"keyword" sql:"IF NOT EXISTS"` + name AccountObjectIdentifier `ddl:"identifier"` + StorageLocations []ExternalVolumeStorageLocation `ddl:"parameter,parentheses" sql:"STORAGE_LOCATIONS"` + AllowWrites *bool `ddl:"parameter" sql:"ALLOW_WRITES"` + Comment *string `ddl:"parameter,single_quotes" sql:"COMMENT"` +} +type ExternalVolumeStorageLocation struct { + S3StorageLocationParams *S3StorageLocationParams `ddl:"list,parentheses,no_comma"` + GCSStorageLocationParams *GCSStorageLocationParams `ddl:"list,parentheses,no_comma"` + AzureStorageLocationParams *AzureStorageLocationParams `ddl:"list,parentheses,no_comma"` +} +type S3StorageLocationParams struct { + Name string `ddl:"parameter,single_quotes" sql:"NAME"` + StorageProvider S3StorageProvider `ddl:"parameter,single_quotes" sql:"STORAGE_PROVIDER"` + StorageAwsRoleArn string `ddl:"parameter,single_quotes" sql:"STORAGE_AWS_ROLE_ARN"` + StorageBaseUrl string `ddl:"parameter,single_quotes" sql:"STORAGE_BASE_URL"` + StorageAwsExternalId *string `ddl:"parameter,single_quotes" sql:"STORAGE_AWS_EXTERNAL_ID"` + Encryption ExternalVolumeS3Encryption `ddl:"list,parentheses,no_comma" sql:"ENCRYPTION ="` +} +type ExternalVolumeS3Encryption struct { + Type S3EncryptionType `ddl:"parameter,single_quotes" sql:"TYPE"` + KmsKeyId *string `ddl:"parameter,single_quotes" sql:"KMS_KEY_ID"` +} +type GCSStorageLocationParams struct { + Name string `ddl:"parameter,single_quotes" sql:"NAME"` + StorageProviderGcs string `ddl:"static" sql:"STORAGE_PROVIDER = 'GCS'"` + StorageBaseUrl string `ddl:"parameter,single_quotes" sql:"STORAGE_BASE_URL"` + Encryption ExternalVolumeGCSEncryption `ddl:"list,parentheses,no_comma" sql:"ENCRYPTION ="` +} +type ExternalVolumeGCSEncryption struct { + Type GCSEncryptionType `ddl:"parameter,single_quotes" sql:"TYPE"` + KmsKeyId *string `ddl:"parameter,single_quotes" sql:"KMS_KEY_ID"` +} +type AzureStorageLocationParams struct { + Name string `ddl:"parameter,single_quotes" sql:"NAME"` + StorageProviderAzure string `ddl:"static" sql:"STORAGE_PROVIDER = 'AZURE'"` + AzureTenantId string `ddl:"parameter,single_quotes" sql:"AZURE_TENANT_ID"` + StorageBaseUrl string `ddl:"parameter,single_quotes" sql:"STORAGE_BASE_URL"` +} + +// AlterExternalVolumeOptions is based on https://docs.snowflake.com/en/sql-reference/sql/alter-external-volume. +type AlterExternalVolumeOptions struct { + alter bool `ddl:"static" sql:"ALTER"` + externalVolume bool `ddl:"static" sql:"EXTERNAL VOLUME"` + IfExists *bool `ddl:"keyword" sql:"IF EXISTS"` + name AccountObjectIdentifier `ddl:"identifier"` + RemoveStorageLocation *string `ddl:"parameter,single_quotes,no_equals" sql:"REMOVE STORAGE_LOCATION"` + Set *AlterExternalVolumeSet `ddl:"keyword" sql:"SET"` + AddStorageLocation *ExternalVolumeStorageLocation `ddl:"parameter" sql:"ADD STORAGE_LOCATION"` +} +type AlterExternalVolumeSet struct { + AllowWrites *bool `ddl:"parameter" sql:"ALLOW_WRITES"` + Comment *string `ddl:"parameter,single_quotes" sql:"COMMENT"` +} + +// DropExternalVolumeOptions is based on https://docs.snowflake.com/en/sql-reference/sql/drop-external-volume. +type DropExternalVolumeOptions struct { + drop bool `ddl:"static" sql:"DROP"` + externalVolume bool `ddl:"static" sql:"EXTERNAL VOLUME"` + IfExists *bool `ddl:"keyword" sql:"IF EXISTS"` + name AccountObjectIdentifier `ddl:"identifier"` +} + +// DescribeExternalVolumeOptions is based on https://docs.snowflake.com/en/sql-reference/sql/desc-external-volume. +type DescribeExternalVolumeOptions struct { + describe bool `ddl:"static" sql:"DESCRIBE"` + externalVolume bool `ddl:"static" sql:"EXTERNAL VOLUME"` + name AccountObjectIdentifier `ddl:"identifier"` +} +type externalVolumeDescRow struct { + ParentProperty string `db:"parent_property"` + Property string `db:"property"` + PropertyType string `db:"property_type"` + PropertyValue string `db:"property_value"` + PropertyDefault string `db:"property_default"` +} +type ExternalVolumeProperty struct { + Parent string + Name string + Type string + Value string + Default string +} + +// ShowExternalVolumeOptions is based on https://docs.snowflake.com/en/sql-reference/sql/show-external-volumes. +type ShowExternalVolumeOptions struct { + show bool `ddl:"static" sql:"SHOW"` + externalVolumes bool `ddl:"static" sql:"EXTERNAL VOLUMES"` + Like *Like `ddl:"keyword" sql:"LIKE"` +} +type externalVolumeShowRow struct { + Name string `db:"name"` + AllowWrites string `db:"allow_writes"` + Comment string `db:"comment"` +} +type ExternalVolume struct { + Name string + AllowWrites string + Comment string +} diff --git a/pkg/sdk/external_volumes_gen_test.go b/pkg/sdk/external_volumes_gen_test.go new file mode 100644 index 0000000000..3edc069b80 --- /dev/null +++ b/pkg/sdk/external_volumes_gen_test.go @@ -0,0 +1,391 @@ +package sdk + +import "testing" + +// Storage location structs for testing +var s3StorageLocationParams = &S3StorageLocationParams{ + Name: "some s3 name", + StorageProvider: S3StorageProviderS3, + StorageAwsRoleArn: "some s3 role arn", + StorageBaseUrl: "some s3 base url", + Encryption: ExternalVolumeS3Encryption{ + Type: S3EncryptionTypeSseS3, + KmsKeyId: String("some s3 kms key id"), + }, +} + +var s3StorageLocationParamsNoEncryption = &S3StorageLocationParams{ + Name: "some s3 name", + StorageProvider: S3StorageProviderS3, + StorageAwsRoleArn: "some s3 role arn", + StorageBaseUrl: "some s3 base url", + Encryption: ExternalVolumeS3Encryption{ + Type: S3EncryptionNone, + }, +} + +var s3StorageLocationParamsGov = &S3StorageLocationParams{ + Name: "some s3 name", + StorageProvider: S3StorageProviderS3GOV, + StorageAwsRoleArn: "some s3 role arn", + StorageBaseUrl: "some s3 base url", + Encryption: ExternalVolumeS3Encryption{ + Type: S3EncryptionTypeSseS3, + KmsKeyId: String("some s3 kms key id"), + }, +} + +var s3StorageLocationParamsWithExternalId = &S3StorageLocationParams{ + Name: "some s3 name", + StorageProvider: S3StorageProviderS3, + StorageAwsRoleArn: "some s3 role arn", + StorageBaseUrl: "some s3 base url", + StorageAwsExternalId: String("some s3 external id"), + Encryption: ExternalVolumeS3Encryption{ + Type: S3EncryptionTypeSseS3, + KmsKeyId: String("some s3 kms key id"), + }, +} + +var gcsStorageLocationParams = &GCSStorageLocationParams{ + Name: "some gcs name", + StorageBaseUrl: "some gcs base url", + Encryption: ExternalVolumeGCSEncryption{ + Type: GCSEncryptionTypeSseKms, + KmsKeyId: String("some gcs kms key id"), + }, +} + +var gcsStorageLocationParamsNoEncryption = &GCSStorageLocationParams{ + Name: "some gcs name", + StorageBaseUrl: "some gcs base url", + Encryption: ExternalVolumeGCSEncryption{ + Type: GCSEncryptionTypeNone, + }, +} + +var azureStorageLocationParams = &AzureStorageLocationParams{ + Name: "some azure name", + AzureTenantId: "some azure tenant id", + StorageBaseUrl: "some azure base url", +} + + +func TestExternalVolumes_Create(t *testing.T) { + + id := randomAccountObjectIdentifier() + // Minimal valid CreateExternalVolumeOptions + defaultOpts := func() *CreateExternalVolumeOptions { + return &CreateExternalVolumeOptions{ + name: id, + } + } + + t.Run("validation: nil options", func(t *testing.T) { + var opts *CreateExternalVolumeOptions = nil + assertOptsInvalidJoinedErrors(t, opts, ErrNilOptions) + }) + + t.Run("validation: conflicting fields for [opts.OrReplace opts.IfNotExists]", func(t *testing.T) { + opts := defaultOpts() + opts.OrReplace = Bool(true) + opts.IfNotExists = Bool(true) + assertOptsInvalidJoinedErrors(t, opts, errOneOf("CreateExternalVolumeOptions", "OrReplace", "IfNotExists")) + }) + + t.Run("validation: valid identifier for [opts.name]", func(t *testing.T) { + opts := defaultOpts() + opts.name = emptyAccountObjectIdentifier + assertOptsInvalidJoinedErrors(t, opts, ErrInvalidObjectIdentifier) + }) + + t.Run("validation: exactly one field from [opts.StorageLocations[i].S3StorageLocationParams opts.StorageLocations[i].GCSStorageLocationParams opts.StorageLocations[i].AzureStorageLocationParams] should be present", func(t *testing.T) { + opts := defaultOpts() + opts.StorageLocations = []ExternalVolumeStorageLocation{ + { S3StorageLocationParams: s3StorageLocationParams, }, + {}, + { S3StorageLocationParams: s3StorageLocationParams, GCSStorageLocationParams: gcsStorageLocationParams, }, + { S3StorageLocationParams: s3StorageLocationParams, AzureStorageLocationParams: azureStorageLocationParams, }, + { GCSStorageLocationParams: gcsStorageLocationParams, AzureStorageLocationParams: azureStorageLocationParams, }, + { + S3StorageLocationParams: s3StorageLocationParams, + GCSStorageLocationParams: gcsStorageLocationParams, + AzureStorageLocationParams: azureStorageLocationParams, + }, + } + assertOptsInvalidJoinedErrors( + t, + opts, + errExactlyOneOf("CreateExternalVolumeOptions.StorageLocation[1]", "S3StorageLocationParams", "GCSStorageLocationParams", "AzureStorageLocationParams"), + errExactlyOneOf("CreateExternalVolumeOptions.StorageLocation[2]", "S3StorageLocationParams", "GCSStorageLocationParams", "AzureStorageLocationParams"), + errExactlyOneOf("CreateExternalVolumeOptions.StorageLocation[3]", "S3StorageLocationParams", "GCSStorageLocationParams", "AzureStorageLocationParams"), + errExactlyOneOf("CreateExternalVolumeOptions.StorageLocation[4]", "S3StorageLocationParams", "GCSStorageLocationParams", "AzureStorageLocationParams"), + errExactlyOneOf("CreateExternalVolumeOptions.StorageLocation[5]", "S3StorageLocationParams", "GCSStorageLocationParams", "AzureStorageLocationParams"), + ) + }) + + t.Run("validation: length of opts.StorageLocations is > 0", func(t *testing.T) { + opts := defaultOpts() + opts.StorageLocations = []ExternalVolumeStorageLocation{} + assertOptsInvalidJoinedErrors(t, opts, errNotSet("CreateExternalVolumeOptions", "StorageLocations")) + }) + + t.Run("1 storage location - s3", func(t *testing.T) { + opts := defaultOpts() + opts.StorageLocations = []ExternalVolumeStorageLocation{ { S3StorageLocationParams: s3StorageLocationParams, }, } + assertOptsValidAndSQLEquals(t, opts, `CREATE EXTERNAL VOLUME %s STORAGE_LOCATIONS = ((NAME = 'some s3 name' STORAGE_PROVIDER = 'S3' STORAGE_AWS_ROLE_ARN = 'some s3 role arn' STORAGE_BASE_URL = 'some s3 base url' ENCRYPTION = (TYPE = 'AWS_SSE_S3' KMS_KEY_ID = 'some s3 kms key id')))`, id.FullyQualifiedName()) + }) + + t.Run("1 storage location with comment - s3gov", func(t *testing.T) { + opts := defaultOpts() + opts.StorageLocations = []ExternalVolumeStorageLocation{ { S3StorageLocationParams: s3StorageLocationParamsGov, }, } + opts.Comment = String("some comment") + assertOptsValidAndSQLEquals(t, opts, `CREATE EXTERNAL VOLUME %s STORAGE_LOCATIONS = ((NAME = 'some s3 name' STORAGE_PROVIDER = 'S3GOV' STORAGE_AWS_ROLE_ARN = 'some s3 role arn' STORAGE_BASE_URL = 'some s3 base url' ENCRYPTION = (TYPE = 'AWS_SSE_S3' KMS_KEY_ID = 'some s3 kms key id'))) COMMENT = 'some comment'`, id.FullyQualifiedName()) + }) + + t.Run("1 storage location - s3 no encryption", func(t *testing.T) { + opts := defaultOpts() + opts.StorageLocations = []ExternalVolumeStorageLocation{ { S3StorageLocationParams: s3StorageLocationParamsNoEncryption, }, } + assertOptsValidAndSQLEquals(t, opts, `CREATE EXTERNAL VOLUME %s STORAGE_LOCATIONS = ((NAME = 'some s3 name' STORAGE_PROVIDER = 'S3' STORAGE_AWS_ROLE_ARN = 'some s3 role arn' STORAGE_BASE_URL = 'some s3 base url' ENCRYPTION = (TYPE = 'NONE')))`, id.FullyQualifiedName()) + }) + + t.Run("1 storage location with allow writes - gcs", func(t *testing.T) { + opts := defaultOpts() + opts.StorageLocations = []ExternalVolumeStorageLocation{ { GCSStorageLocationParams: gcsStorageLocationParams, }, } + opts.AllowWrites = Bool(true) + assertOptsValidAndSQLEquals(t, opts, `CREATE EXTERNAL VOLUME %s STORAGE_LOCATIONS = ((NAME = 'some gcs name' STORAGE_PROVIDER = 'GCS' STORAGE_BASE_URL = 'some gcs base url' ENCRYPTION = (TYPE = 'GCS_SSE_KMS' KMS_KEY_ID = 'some gcs kms key id'))) ALLOW_WRITES = true`, id.FullyQualifiedName()) + }) + + t.Run("1 storage location with allow writes - gcs no encryption", func(t *testing.T) { + opts := defaultOpts() + opts.StorageLocations = []ExternalVolumeStorageLocation{ { GCSStorageLocationParams: gcsStorageLocationParamsNoEncryption, }, } + opts.AllowWrites = Bool(true) + assertOptsValidAndSQLEquals(t, opts, `CREATE EXTERNAL VOLUME %s STORAGE_LOCATIONS = ((NAME = 'some gcs name' STORAGE_PROVIDER = 'GCS' STORAGE_BASE_URL = 'some gcs base url' ENCRYPTION = (TYPE = 'NONE'))) ALLOW_WRITES = true`, id.FullyQualifiedName()) + }) + + t.Run("1 storage location - azure", func(t *testing.T) { + opts := defaultOpts() + opts.StorageLocations = []ExternalVolumeStorageLocation{ { AzureStorageLocationParams: azureStorageLocationParams, }, } + assertOptsValidAndSQLEquals(t, opts, `CREATE EXTERNAL VOLUME %s STORAGE_LOCATIONS = ((NAME = 'some azure name' STORAGE_PROVIDER = 'AZURE' AZURE_TENANT_ID = 'some azure tenant id' STORAGE_BASE_URL = 'some azure base url'))`, id.FullyQualifiedName()) + }) + + t.Run("3 storage locations and all options", func(t *testing.T) { + opts := defaultOpts() + opts.OrReplace = Bool(true) + opts.StorageLocations = []ExternalVolumeStorageLocation{ + { S3StorageLocationParams: s3StorageLocationParamsWithExternalId, }, + { GCSStorageLocationParams: gcsStorageLocationParams, }, + { AzureStorageLocationParams: azureStorageLocationParams, }, + } + opts.AllowWrites = Bool(false) + opts.Comment = String("some comment") + assertOptsValidAndSQLEquals(t, opts, `CREATE OR REPLACE EXTERNAL VOLUME %s STORAGE_LOCATIONS = ((NAME = 'some s3 name' STORAGE_PROVIDER = 'S3' STORAGE_AWS_ROLE_ARN = 'some s3 role arn' STORAGE_BASE_URL = 'some s3 base url' STORAGE_AWS_EXTERNAL_ID = 'some s3 external id' ENCRYPTION = (TYPE = 'AWS_SSE_S3' KMS_KEY_ID = 'some s3 kms key id')), (NAME = 'some gcs name' STORAGE_PROVIDER = 'GCS' STORAGE_BASE_URL = 'some gcs base url' ENCRYPTION = (TYPE = 'GCS_SSE_KMS' KMS_KEY_ID = 'some gcs kms key id')), (NAME = 'some azure name' STORAGE_PROVIDER = 'AZURE' AZURE_TENANT_ID = 'some azure tenant id' STORAGE_BASE_URL = 'some azure base url')) ALLOW_WRITES = false COMMENT = 'some comment'`, id.FullyQualifiedName()) + }) +} + +func TestExternalVolumes_Alter(t *testing.T) { + + id := randomAccountObjectIdentifier() + // Minimal valid AlterExternalVolumeOptions + defaultOpts := func() *AlterExternalVolumeOptions { + return &AlterExternalVolumeOptions{ + name: id, + } + } + + t.Run("validation: nil options", func(t *testing.T) { + var opts *AlterExternalVolumeOptions = nil + assertOptsInvalidJoinedErrors(t, opts, ErrNilOptions) + }) + + t.Run("validation: exactly one field from [opts.RemoveStorageLocation opts.Set opts.AddStorageLocation] should be present - two set", func(t *testing.T) { + removeAndSetOpts := defaultOpts() + removeAndAddOpts := defaultOpts() + setAndAddOpts := defaultOpts() + + removeAndSetOpts.RemoveStorageLocation = String("some storage location") + removeAndSetOpts.Set = &AlterExternalVolumeSet{ AllowWrites: Bool(true), } + + removeAndAddOpts.RemoveStorageLocation = String("some storage location") + removeAndAddOpts.AddStorageLocation = &ExternalVolumeStorageLocation{ S3StorageLocationParams: s3StorageLocationParamsWithExternalId, } + + setAndAddOpts.Set = &AlterExternalVolumeSet{ AllowWrites: Bool(true), } + setAndAddOpts.AddStorageLocation = &ExternalVolumeStorageLocation{ S3StorageLocationParams: s3StorageLocationParamsWithExternalId, } + + assertOptsInvalidJoinedErrors(t, removeAndSetOpts, errExactlyOneOf("AlterExternalVolumeOptions", "RemoveStorageLocation", "Set", "AddStorageLocation")) + assertOptsInvalidJoinedErrors(t, removeAndAddOpts, errExactlyOneOf("AlterExternalVolumeOptions", "RemoveStorageLocation", "Set", "AddStorageLocation")) + assertOptsInvalidJoinedErrors(t, setAndAddOpts, errExactlyOneOf("AlterExternalVolumeOptions", "RemoveStorageLocation", "Set", "AddStorageLocation")) + }) + + t.Run("validation: exactly one field from [opts.RemoveStorageLocation opts.Set opts.AddStorageLocation] should be present - three set", func(t *testing.T) { + opts := defaultOpts() + opts.RemoveStorageLocation = String("some storage location") + opts.Set = &AlterExternalVolumeSet{ AllowWrites: Bool(true), } + opts.AddStorageLocation = &ExternalVolumeStorageLocation{ S3StorageLocationParams: s3StorageLocationParamsWithExternalId, } + assertOptsInvalidJoinedErrors(t, opts, errExactlyOneOf("AlterExternalVolumeOptions", "RemoveStorageLocation", "Set", "AddStorageLocation")) + }) + + t.Run("validation: valid identifier for [opts.name]", func(t *testing.T) { + opts := defaultOpts() + opts.name = emptyAccountObjectIdentifier + assertOptsInvalidJoinedErrors(t, opts, ErrInvalidObjectIdentifier) + }) + + t.Run("validation: exactly one field from [opts.AddStorageLocation.S3StorageLocationParams opts.AddStorageLocation.GCSStorageLocationParams opts.AddStorageLocation.AzureStorageLocationParams] should be present - none set", func(t *testing.T) { + opts := defaultOpts() + opts.AddStorageLocation = &ExternalVolumeStorageLocation{} + assertOptsInvalidJoinedErrors(t, opts, errExactlyOneOf("AlterExternalVolumeOptions.AddStorageLocation", "S3StorageLocationParams", "GCSStorageLocationParams", "AzureStorageLocationParams")) + }) + + t.Run("validation: exactly one field from [opts.AddStorageLocation.S3StorageLocationParams opts.AddStorageLocation.GCSStorageLocationParams opts.AddStorageLocation.AzureStorageLocationParams] should be present - two set", func(t *testing.T) { + s3AndGcsOpts := defaultOpts() + s3AndAzureOpts := defaultOpts() + gcsAndAzureOpts := defaultOpts() + s3AndGcsOpts.AddStorageLocation = &ExternalVolumeStorageLocation{ + S3StorageLocationParams: s3StorageLocationParams, + GCSStorageLocationParams: gcsStorageLocationParams, + } + s3AndAzureOpts.AddStorageLocation = &ExternalVolumeStorageLocation{ + S3StorageLocationParams: s3StorageLocationParams, + AzureStorageLocationParams: azureStorageLocationParams, + } + gcsAndAzureOpts.AddStorageLocation = &ExternalVolumeStorageLocation{ + GCSStorageLocationParams: gcsStorageLocationParams, + AzureStorageLocationParams: azureStorageLocationParams, + } + assertOptsInvalidJoinedErrors(t, s3AndGcsOpts, errExactlyOneOf("AlterExternalVolumeOptions.AddStorageLocation", "S3StorageLocationParams", "GCSStorageLocationParams", "AzureStorageLocationParams")) + assertOptsInvalidJoinedErrors(t, s3AndAzureOpts, errExactlyOneOf("AlterExternalVolumeOptions.AddStorageLocation", "S3StorageLocationParams", "GCSStorageLocationParams", "AzureStorageLocationParams")) + assertOptsInvalidJoinedErrors(t, gcsAndAzureOpts, errExactlyOneOf("AlterExternalVolumeOptions.AddStorageLocation", "S3StorageLocationParams", "GCSStorageLocationParams", "AzureStorageLocationParams")) + }) + + t.Run("validation: exactly one field from [opts.AddStorageLocation.S3StorageLocationParams opts.AddStorageLocation.GCSStorageLocationParams opts.AddStorageLocation.AzureStorageLocationParams] should be present - three set", func(t *testing.T) { + opts := defaultOpts() + opts.AddStorageLocation = &ExternalVolumeStorageLocation{ + S3StorageLocationParams: s3StorageLocationParams, + GCSStorageLocationParams: gcsStorageLocationParams, + AzureStorageLocationParams: azureStorageLocationParams, + } + assertOptsInvalidJoinedErrors(t, opts, errExactlyOneOf("AlterExternalVolumeOptions.AddStorageLocation", "S3StorageLocationParams", "GCSStorageLocationParams", "AzureStorageLocationParams")) + }) + + t.Run("remove storage location", func(t *testing.T) { + opts := defaultOpts() + opts.RemoveStorageLocation = String("some storage location") + assertOptsValidAndSQLEquals(t, opts, `ALTER EXTERNAL VOLUME %s REMOVE STORAGE_LOCATION 'some storage location'`, id.FullyQualifiedName()) + }) + + t.Run("set - allow writes", func(t *testing.T) { + opts := defaultOpts() + opts.Set = &AlterExternalVolumeSet{ AllowWrites: Bool(true), } + assertOptsValidAndSQLEquals(t, opts, `ALTER EXTERNAL VOLUME %s SET ALLOW_WRITES = true`, id.FullyQualifiedName()) + }) + + t.Run("set - comment", func(t *testing.T) { + opts := defaultOpts() + opts.Set = &AlterExternalVolumeSet{ Comment: String("some comment"), } + assertOptsValidAndSQLEquals(t, opts, `ALTER EXTERNAL VOLUME %s SET COMMENT = 'some comment'`, id.FullyQualifiedName()) + }) + + t.Run("add storage location - s3", func(t *testing.T) { + opts := defaultOpts() + opts.AddStorageLocation = &ExternalVolumeStorageLocation{ S3StorageLocationParams: s3StorageLocationParamsWithExternalId, } + assertOptsValidAndSQLEquals(t, opts, `ALTER EXTERNAL VOLUME %s ADD STORAGE_LOCATION = (NAME = 'some s3 name' STORAGE_PROVIDER = 'S3' STORAGE_AWS_ROLE_ARN = 'some s3 role arn' STORAGE_BASE_URL = 'some s3 base url' STORAGE_AWS_EXTERNAL_ID = 'some s3 external id' ENCRYPTION = (TYPE = 'AWS_SSE_S3' KMS_KEY_ID = 'some s3 kms key id'))`, id.FullyQualifiedName()) + }) + + t.Run("add storage location - gcs", func(t *testing.T) { + opts := defaultOpts() + opts.AddStorageLocation = &ExternalVolumeStorageLocation{ GCSStorageLocationParams: gcsStorageLocationParams, } + assertOptsValidAndSQLEquals(t, opts, `ALTER EXTERNAL VOLUME %s ADD STORAGE_LOCATION = (NAME = 'some gcs name' STORAGE_PROVIDER = 'GCS' STORAGE_BASE_URL = 'some gcs base url' ENCRYPTION = (TYPE = 'GCS_SSE_KMS' KMS_KEY_ID = 'some gcs kms key id'))`, id.FullyQualifiedName()) + }) + + t.Run("add storage location - azure", func(t *testing.T) { + opts := defaultOpts() + opts.AddStorageLocation = &ExternalVolumeStorageLocation{ AzureStorageLocationParams: azureStorageLocationParams, } + assertOptsValidAndSQLEquals(t, opts, `ALTER EXTERNAL VOLUME %s ADD STORAGE_LOCATION = (NAME = 'some azure name' STORAGE_PROVIDER = 'AZURE' AZURE_TENANT_ID = 'some azure tenant id' STORAGE_BASE_URL = 'some azure base url')`, id.FullyQualifiedName()) + }) +} + +func TestExternalVolumes_Drop(t *testing.T) { + + id := randomAccountObjectIdentifier() + // Minimal valid DropExternalVolumeOptions + defaultOpts := func() *DropExternalVolumeOptions { + return &DropExternalVolumeOptions{ + name: id, + } + } + + t.Run("validation: nil options", func(t *testing.T) { + var opts *DropExternalVolumeOptions = nil + assertOptsInvalidJoinedErrors(t, opts, ErrNilOptions) + }) + + t.Run("validation: valid identifier for [opts.name]", func(t *testing.T) { + opts := defaultOpts() + opts.name = emptyAccountObjectIdentifier + assertOptsInvalidJoinedErrors(t, opts, ErrInvalidObjectIdentifier) + }) + + t.Run("basic", func(t *testing.T) { + opts := defaultOpts() + assertOptsValidAndSQLEquals(t, opts, `DROP EXTERNAL VOLUME %s`, id.FullyQualifiedName()) + }) + + t.Run("all options", func(t *testing.T) { + opts := defaultOpts() + opts.IfExists = Bool(true) + assertOptsValidAndSQLEquals(t, opts, `DROP EXTERNAL VOLUME IF EXISTS %s`, id.FullyQualifiedName()) + }) +} + +func TestExternalVolumes_Describe(t *testing.T) { + + id := randomAccountObjectIdentifier() + // Minimal valid DescribeExternalVolumeOptions + defaultOpts := func() *DescribeExternalVolumeOptions { + return &DescribeExternalVolumeOptions{ + name: id, + } + } + + t.Run("validation: nil options", func(t *testing.T) { + var opts *DescribeExternalVolumeOptions = nil + assertOptsInvalidJoinedErrors(t, opts, ErrNilOptions) + }) + t.Run("validation: valid identifier for [opts.name]", func(t *testing.T) { + opts := defaultOpts() + opts.name = emptyAccountObjectIdentifier + assertOptsInvalidJoinedErrors(t, opts, ErrInvalidObjectIdentifier) + }) + + t.Run("basic", func(t *testing.T) { + opts := defaultOpts() + assertOptsValidAndSQLEquals(t, opts, `DESCRIBE EXTERNAL VOLUME %s`, id.FullyQualifiedName()) + }) +} + +func TestExternalVolumes_Show(t *testing.T) { + id := randomAccountObjectIdentifier() + // Minimal valid ShowExternalVolumeOptions + defaultOpts := func() *ShowExternalVolumeOptions { + return &ShowExternalVolumeOptions{} + } + + t.Run("validation: nil options", func(t *testing.T) { + var opts *ShowExternalVolumeOptions = nil + assertOptsInvalidJoinedErrors(t, opts, ErrNilOptions) + }) + + t.Run("basic", func(t *testing.T) { + opts := defaultOpts() + assertOptsValidAndSQLEquals(t, opts, "SHOW EXTERNAL VOLUMES") + }) + + t.Run("all options", func(t *testing.T) { + opts := defaultOpts() + opts.Like = &Like{ + Pattern: String(id.Name()), + } + assertOptsValidAndSQLEquals(t, opts, "SHOW EXTERNAL VOLUMES LIKE '%s'", id.Name()) + }) +} diff --git a/pkg/sdk/external_volumes_impl_gen.go b/pkg/sdk/external_volumes_impl_gen.go new file mode 100644 index 0000000000..459fced135 --- /dev/null +++ b/pkg/sdk/external_volumes_impl_gen.go @@ -0,0 +1,174 @@ +package sdk + +import ( + "context" + "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/internal/collections" +) + +var _ ExternalVolumes = (*externalVolumes)(nil) + +type externalVolumes struct { + client *Client +} + +func (v *externalVolumes) Create(ctx context.Context, request *CreateExternalVolumeRequest) error { + opts := request.toOpts() + return validateAndExec(v.client, ctx, opts) +} + +func (v *externalVolumes) Alter(ctx context.Context, request *AlterExternalVolumeRequest) error { + opts := request.toOpts() + return validateAndExec(v.client, ctx, opts) +} + +func (v *externalVolumes) Drop(ctx context.Context, request *DropExternalVolumeRequest) error { + opts := request.toOpts() + return validateAndExec(v.client, ctx, opts) +} + +func (v *externalVolumes) Describe(ctx context.Context, id AccountObjectIdentifier) ([]ExternalVolumeProperty, error) { + opts := &DescribeExternalVolumeOptions{ + name: id, + } + rows, err := validateAndQuery[externalVolumeDescRow](v.client, ctx, opts) + if err != nil { + return nil, err + } + return convertRows[externalVolumeDescRow, ExternalVolumeProperty](rows), nil +} + +func (v *externalVolumes) Show(ctx context.Context, request *ShowExternalVolumeRequest) ([]ExternalVolume, error) { + opts := request.toOpts() + dbRows, err := validateAndQuery[externalVolumeShowRow](v.client, ctx, opts) + if err != nil { + return nil, err + } + resultList := convertRows[externalVolumeShowRow, ExternalVolume](dbRows) + return resultList, nil +} + +func (v *externalVolumes) ShowByID(ctx context.Context, id AccountObjectIdentifier) (*ExternalVolume, error) { + externalVolumes, err := v.Show(ctx, NewShowExternalVolumeRequest()) + if err != nil { + return nil, err + } + return collections.FindFirst(externalVolumes, func(r ExternalVolume) bool { return r.Name == id.Name() }) +} + +func (r *CreateExternalVolumeRequest) toOpts() *CreateExternalVolumeOptions { + opts := &CreateExternalVolumeOptions{ + OrReplace: r.OrReplace, + IfNotExists: r.IfNotExists, + name: r.name, + StorageLocations: r.StorageLocations, + AllowWrites: r.AllowWrites, + Comment: r.Comment, + } + return opts +} + +func (r *AlterExternalVolumeRequest) toOpts() *AlterExternalVolumeOptions { + opts := &AlterExternalVolumeOptions{ + IfExists: r.IfExists, + name: r.name, + RemoveStorageLocation: r.RemoveStorageLocation, + } + + if r.Set != nil { + + opts.Set = &AlterExternalVolumeSet{ + AllowWrites: r.Set.AllowWrites, + Comment: r.Set.Comment, + } + + } + + if r.AddStorageLocation != nil { + + opts.AddStorageLocation = &ExternalVolumeStorageLocation{} + + if r.AddStorageLocation.S3StorageLocationParams != nil { + + opts.AddStorageLocation.S3StorageLocationParams = &S3StorageLocationParams{ + Name: r.AddStorageLocation.S3StorageLocationParams.Name, + StorageProvider: r.AddStorageLocation.S3StorageLocationParams.StorageProvider, + StorageAwsRoleArn: r.AddStorageLocation.S3StorageLocationParams.StorageAwsRoleArn, + StorageBaseUrl: r.AddStorageLocation.S3StorageLocationParams.StorageBaseUrl, + StorageAwsExternalId: r.AddStorageLocation.S3StorageLocationParams.StorageAwsExternalId, + } + + opts.AddStorageLocation.S3StorageLocationParams.Encryption = ExternalVolumeS3Encryption{ + Type: r.AddStorageLocation.S3StorageLocationParams.Encryption.Type, + KmsKeyId: r.AddStorageLocation.S3StorageLocationParams.Encryption.KmsKeyId, + } + + } + + if r.AddStorageLocation.GCSStorageLocationParams != nil { + + opts.AddStorageLocation.GCSStorageLocationParams = &GCSStorageLocationParams{ + Name: r.AddStorageLocation.GCSStorageLocationParams.Name, + StorageBaseUrl: r.AddStorageLocation.GCSStorageLocationParams.StorageBaseUrl, + } + + opts.AddStorageLocation.GCSStorageLocationParams.Encryption = ExternalVolumeGCSEncryption{ + Type: r.AddStorageLocation.GCSStorageLocationParams.Encryption.Type, + KmsKeyId: r.AddStorageLocation.GCSStorageLocationParams.Encryption.KmsKeyId, + } + + } + + if r.AddStorageLocation.AzureStorageLocationParams != nil { + + opts.AddStorageLocation.AzureStorageLocationParams = &AzureStorageLocationParams{ + Name: r.AddStorageLocation.AzureStorageLocationParams.Name, + AzureTenantId: r.AddStorageLocation.AzureStorageLocationParams.AzureTenantId, + StorageBaseUrl: r.AddStorageLocation.AzureStorageLocationParams.StorageBaseUrl, + } + + } + + } + + return opts +} + +func (r *DropExternalVolumeRequest) toOpts() *DropExternalVolumeOptions { + opts := &DropExternalVolumeOptions{ + IfExists: r.IfExists, + name: r.name, + } + return opts +} + +func (r *DescribeExternalVolumeRequest) toOpts() *DescribeExternalVolumeOptions { + opts := &DescribeExternalVolumeOptions{ + name: r.name, + } + return opts +} + +func (r externalVolumeDescRow) convert() *ExternalVolumeProperty { + return &ExternalVolumeProperty{ + Parent: r.ParentProperty, + Name: r.Property, + Type: r.PropertyType, + Value: r.PropertyValue, + Default: r.PropertyDefault, + } +} + +func (r *ShowExternalVolumeRequest) toOpts() *ShowExternalVolumeOptions { + opts := &ShowExternalVolumeOptions{ + Like: r.Like, + } + return opts +} + +func (r externalVolumeShowRow) convert() *ExternalVolume { + return &ExternalVolume{ + Name: r.Name, + AllowWrites: r.AllowWrites, + Comment: r.Comment, + } +} diff --git a/pkg/sdk/external_volumes_validations_gen.go b/pkg/sdk/external_volumes_validations_gen.go new file mode 100644 index 0000000000..248204d0bf --- /dev/null +++ b/pkg/sdk/external_volumes_validations_gen.go @@ -0,0 +1,89 @@ +package sdk + +import "fmt" + +var ( + _ validatable = new(CreateExternalVolumeOptions) + _ validatable = new(AlterExternalVolumeOptions) + _ validatable = new(DropExternalVolumeOptions) + _ validatable = new(DescribeExternalVolumeOptions) + _ validatable = new(ShowExternalVolumeOptions) +) + +func (opts *CreateExternalVolumeOptions) validate() error { + if opts == nil { + return ErrNilOptions + } + var errs []error + if everyValueSet(opts.OrReplace, opts.IfNotExists) { + errs = append(errs, errOneOf("CreateExternalVolumeOptions", "OrReplace", "IfNotExists")) + } + if !ValidObjectIdentifier(opts.name) { + errs = append(errs, ErrInvalidObjectIdentifier) + } + + // Custom (not code generated) validations + + // Apply errExactlyOneOf to each element in storage locations list + for i,storageLocation := range opts.StorageLocations { + if !exactlyOneValueSet(storageLocation.S3StorageLocationParams, storageLocation.GCSStorageLocationParams, storageLocation.AzureStorageLocationParams) { + errs = append(errs, errExactlyOneOf(fmt.Sprintf("CreateExternalVolumeOptions.StorageLocation[%d]", i), "S3StorageLocationParams", "GCSStorageLocationParams", "AzureStorageLocationParams")) + } + } + + // Check the storage location list is not empty, as at least 1 storage location is required for an external volume + if len(opts.StorageLocations) == 0 { + errs = append(errs, errNotSet("CreateExternalVolumeOptions", "StorageLocations")) + } + + return JoinErrors(errs...) +} + +func (opts *AlterExternalVolumeOptions) validate() error { + if opts == nil { + return ErrNilOptions + } + var errs []error + if !exactlyOneValueSet(opts.RemoveStorageLocation, opts.Set, opts.AddStorageLocation) { + errs = append(errs, errExactlyOneOf("AlterExternalVolumeOptions", "RemoveStorageLocation", "Set", "AddStorageLocation")) + } + if !ValidObjectIdentifier(opts.name) { + errs = append(errs, ErrInvalidObjectIdentifier) + } + if valueSet(opts.AddStorageLocation) { + if !exactlyOneValueSet(opts.AddStorageLocation.S3StorageLocationParams, opts.AddStorageLocation.GCSStorageLocationParams, opts.AddStorageLocation.AzureStorageLocationParams) { + errs = append(errs, errExactlyOneOf("AlterExternalVolumeOptions.AddStorageLocation", "S3StorageLocationParams", "GCSStorageLocationParams", "AzureStorageLocationParams")) + } + } + return JoinErrors(errs...) +} + +func (opts *DropExternalVolumeOptions) validate() error { + if opts == nil { + return ErrNilOptions + } + var errs []error + if !ValidObjectIdentifier(opts.name) { + errs = append(errs, ErrInvalidObjectIdentifier) + } + return JoinErrors(errs...) +} + +func (opts *DescribeExternalVolumeOptions) validate() error { + if opts == nil { + return ErrNilOptions + } + var errs []error + if !ValidObjectIdentifier(opts.name) { + errs = append(errs, ErrInvalidObjectIdentifier) + } + return JoinErrors(errs...) +} + +func (opts *ShowExternalVolumeOptions) validate() error { + if opts == nil { + return ErrNilOptions + } + var errs []error + return JoinErrors(errs...) +} diff --git a/pkg/sdk/poc/main.go b/pkg/sdk/poc/main.go index e62a113123..11ed5092b7 100644 --- a/pkg/sdk/poc/main.go +++ b/pkg/sdk/poc/main.go @@ -43,6 +43,7 @@ var definitionMapping = map[string]*generator.Interface{ "security_integrations_def.go": sdk.SecurityIntegrationsDef, "cortex_search_services_def.go": sdk.CortexSearchServiceDef, "data_metric_function_references_def.go": sdk.DataMetricFunctionReferenceDef, + "external_volumes_def.go": sdk.ExternalVolumesDef, } func main() { diff --git a/pkg/sdk/testint/external_volumes_gen_integration_test.go b/pkg/sdk/testint/external_volumes_gen_integration_test.go new file mode 100644 index 0000000000..7a16924088 --- /dev/null +++ b/pkg/sdk/testint/external_volumes_gen_integration_test.go @@ -0,0 +1,566 @@ +package testint + +import ( + "testing" + "strconv" + "strings" + "fmt" + "encoding/json" + "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/acceptance/testenvs" + "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/sdk" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Cannot test awsgov on non-gov Snowflake deployments + +func TestInt_ExternalVolumes(t *testing.T) { + client := testClient(t) + ctx := testContext(t) + + awsBaseUrl := testenvs.GetOrSkipTest(t, testenvs.AwsExternalBucketUrl) + awsRoleARN := testenvs.GetOrSkipTest(t, testenvs.AwsExternalRoleArn) + awsKmsKeyId := testenvs.GetOrSkipTest(t, testenvs.AwsExternalKeyId) + awsExternalId := "123456789" + + gcsBaseUrl := testenvs.GetOrSkipTest(t, testenvs.GcsExternalBuckerUrl) + // //TODO - To test external volumes wth gcs storage locations that have encryption TYPE=GCS_SSE_KMS a gcs KMS_KEY_ID is required, which is not currently available in testenvs + + azureBaseUrl := testenvs.GetOrSkipTest(t, testenvs.AzureExternalBucketUrl) // TODO verify assumption on form + azureTenantId := testenvs.GetOrSkipTest(t, testenvs.AzureExternalTenantId) + + assertExternalVolumeShowResult := func(t *testing.T, s *sdk.ExternalVolume, name sdk.AccountObjectIdentifier, allowWrites bool, comment string) { + t.Helper() + assert.Equal(t, name.Name(), s.Name) + assert.Equal(t, strconv.FormatBool(allowWrites), s.AllowWrites) + assert.Equal(t, comment, s.Comment) + } + + type ExternalVolumePropNameValue struct { + Name string + Value string + } + + type S3StorageLocation struct { + Name string `json:"NAME"` + StorageProvider string `json:"STORAGE_PROVIDER"` + StorageBaseUrl string `json:"STORAGE_BASE_URL"` + StorageAllowedLocations []string `json:"STORAGE_ALLOWED_LOCATIONS"` + StorageAwsRoleArn string `json:"STORAGE_AWS_ROLE_ARN"` + StroageAwsIamUserArn string `json:"STORAGE_AWS_IAM_USER_ARN"` + StorageAwsExternalId string `json:"STORAGE_AWS_EXTERNAL_ID"` + EncryptionType string `json:"ENCRYPTION_TYPE"` + EncryptionKmsId string `json:"ENCRYPTION_KMS_KEY_ID"` + } + + type S3StorageLocationTrimmed struct { + Name string `json:"NAME"` + StorageProvider string `json:"STORAGE_PROVIDER"` + StorageBaseUrl string `json:"STORAGE_BASE_URL"` + StorageAwsRoleArn string `json:"STORAGE_AWS_ROLE_ARN"` + StorageAwsExternalId string `json:"STORAGE_AWS_EXTERNAL_ID"` + EncryptionType string `json:"ENCRYPTION_TYPE"` + EncryptionKmsId string `json:"ENCRYPTION_KMS_KEY_ID,omitempty"` + } + + type GCSStorageLocation struct { + Name string `json:"NAME"` + StorageProvider string `json:"STORAGE_PROVIDER"` + StorageBaseUrl string `json:"STORAGE_BASE_URL"` + StorageAllowedLocations []string `json:"STORAGE_ALLOWED_LOCATIONS"` + StorageGcpServiceAccount string `json:"STORAGE_GCP_SERVICE_ACCOUNT"` + EncryptionType string `json:"ENCRYPTION_TYPE"` + EncryptionKmsId string `json:"ENCRYPTION_KMS_KEY_ID"` + } + + type GCSStorageLocationTrimmed struct { + Name string `json:"NAME"` + StorageProvider string `json:"STORAGE_PROVIDER"` + StorageBaseUrl string `json:"STORAGE_BASE_URL"` + EncryptionType string `json:"ENCRYPTION_TYPE"` + EncryptionKmsId string `json:"ENCRYPTION_KMS_KEY_ID,omitempty"` + } + + type AzureStorageLocation struct { + Name string `json:"NAME"` + StorageProvider string `json:"STORAGE_PROVIDER"` + StorageBaseUrl string `json:"STORAGE_BASE_URL"` + StorageAllowedLocations []string `json:"STORAGE_ALLOWED_LOCATIONS"` + AzureTenantId string `json:"AZURE_TENANT_ID"` + AzureMultiTenantAppName string `json:"AZURE_MULTI_TENANT_APP_NAME"` + AzureConsentUrl string `json:"AZURE_CONSENT_URL"` + EncryptionType string `json:"ENCRYPTION_TYPE"` + EncryptionKmsId string `json:"ENCRYPTION_KMS_KEY_ID"` + } + + type AzureStorageLocationTrimmed struct { + Name string `json:"NAME"` + StorageProvider string `json:"STORAGE_PROVIDER"` + StorageBaseUrl string `json:"STORAGE_BASE_URL"` + AzureTenantId string `json:"AZURE_TENANT_ID"` + } + + // Enforce only property names and values in tests, not parent_property, type and property_default + // In addition the storage location properties are trimmed to only contain values that we set + trimProperties := func(t *testing.T, props []sdk.ExternalVolumeProperty) []ExternalVolumePropNameValue { + t.Helper() + var externalVolumePropNameValue []ExternalVolumePropNameValue + for _,p := range props { + if strings.Contains(p.Name, "STORAGE_LOCATION_") { + if strings.Contains(p.Value, `"STORAGE_PROVIDER":"S3"`) { + s3StorageLocation := S3StorageLocation{} + json.Unmarshal([]byte(p.Value), &s3StorageLocation) + s3StorageLocationTrimmed := S3StorageLocationTrimmed{ + Name: s3StorageLocation.Name, + StorageProvider: s3StorageLocation.StorageProvider, + StorageBaseUrl: s3StorageLocation.StorageBaseUrl, + StorageAwsRoleArn: s3StorageLocation.StorageAwsRoleArn, + StorageAwsExternalId: s3StorageLocation.StorageAwsExternalId, + EncryptionType: s3StorageLocation.EncryptionType, + EncryptionKmsId: s3StorageLocation.EncryptionKmsId, + } + s3StorageLocationTrimmedMarshaled, err := json.Marshal(s3StorageLocationTrimmed) + require.NoError(t, err) + externalVolumePropNameValue = append( + externalVolumePropNameValue, + ExternalVolumePropNameValue{Name: p.Name, Value: string(s3StorageLocationTrimmedMarshaled)}, + ) + } else if strings.Contains(p.Value, `"STORAGE_PROVIDER":"GCS"`) { + gcsStorageLocation := GCSStorageLocation{} + json.Unmarshal([]byte(p.Value), &gcsStorageLocation) + gcsStorageLocationTrimmed := GCSStorageLocationTrimmed{ + Name: gcsStorageLocation.Name, + StorageProvider: gcsStorageLocation.StorageProvider, + StorageBaseUrl: gcsStorageLocation.StorageBaseUrl, + EncryptionType: gcsStorageLocation.EncryptionType, + EncryptionKmsId: gcsStorageLocation.EncryptionKmsId, + } + gcsStorageLocationTrimmedMarshaled, err := json.Marshal(gcsStorageLocationTrimmed) + require.NoError(t, err) + externalVolumePropNameValue = append( + externalVolumePropNameValue, + ExternalVolumePropNameValue{Name: p.Name, Value: string(gcsStorageLocationTrimmedMarshaled)}, + ) + } else if strings.Contains(p.Value, `"STORAGE_PROVIDER":"AZURE"`) { + azureStorageLocation := AzureStorageLocation{} + json.Unmarshal([]byte(p.Value), &azureStorageLocation) + azureStorageLocationTrimmed := AzureStorageLocationTrimmed{ + Name: azureStorageLocation.Name, + StorageProvider: azureStorageLocation.StorageProvider, + StorageBaseUrl: azureStorageLocation.StorageBaseUrl, + AzureTenantId: azureStorageLocation.AzureTenantId, + } + azureStorageLocationTrimmedMarshaled, err := json.Marshal(azureStorageLocationTrimmed) + require.NoError(t, err) + externalVolumePropNameValue = append( + externalVolumePropNameValue, + ExternalVolumePropNameValue{Name: p.Name, Value: string(azureStorageLocationTrimmedMarshaled)}, + ) + } else { + panic("Unrecognised storage provider in storage location property") + } + } else { + externalVolumePropNameValue = append(externalVolumePropNameValue, ExternalVolumePropNameValue{Name: p.Name, Value: p.Value}) + } + } + + return externalVolumePropNameValue + } + + s3StorageLocations := []sdk.ExternalVolumeStorageLocation{ + { + S3StorageLocationParams: &sdk.S3StorageLocationParams{ + Name: "s3_testing_storage_location", + StorageProvider: sdk.S3StorageProviderS3, + StorageAwsRoleArn: awsRoleARN, + StorageBaseUrl: awsBaseUrl, + StorageAwsExternalId: sdk.String(awsExternalId), + Encryption: sdk.ExternalVolumeS3Encryption{ + Type: sdk.S3EncryptionTypeSseKms, + KmsKeyId: &awsKmsKeyId, + }, + }, + }, + } + + s3StorageLocationsNoneEncryption := []sdk.ExternalVolumeStorageLocation{ + { + S3StorageLocationParams: &sdk.S3StorageLocationParams{ + Name: "s3_testing_storage_location_no_encryption", + StorageProvider: sdk.S3StorageProviderS3, + StorageAwsRoleArn: awsRoleARN, + StorageBaseUrl: awsBaseUrl, + StorageAwsExternalId: sdk.String(awsExternalId), + Encryption: sdk.ExternalVolumeS3Encryption{ + Type: sdk.S3EncryptionNone, + }, + }, + }, + } + + gcsStorageLocations := []sdk.ExternalVolumeStorageLocation{ + { + GCSStorageLocationParams: &sdk.GCSStorageLocationParams{ + Name: "gcs_testing_storage_location", + StorageBaseUrl: gcsBaseUrl, + Encryption: sdk.ExternalVolumeGCSEncryption{ + Type: sdk.GCSEncryptionTypeNone, + }, + }, + }, + } + + azureStorageLocations := []sdk.ExternalVolumeStorageLocation{ + { + AzureStorageLocationParams: &sdk.AzureStorageLocationParams{ + Name: "azure_testing_storage_location", + AzureTenantId: azureTenantId, + StorageBaseUrl: azureBaseUrl, + }, + }, + } + + createExternalVolume:= func(t *testing.T, storageLocations []sdk.ExternalVolumeStorageLocation, allowWrites bool, comment string) sdk.AccountObjectIdentifier { + t.Helper() + + //id := testClientHelper().Ids.RandomAccountObjectIdentifier() + //TODO remove this + id := sdk.NewAccountObjectIdentifier("test_external_volume") + req := sdk.NewCreateExternalVolumeRequest(id, storageLocations). + WithIfNotExists(true). + WithAllowWrites(allowWrites). + WithComment(comment) + + err := client.ExternalVolumes.Create(ctx, req) + require.NoError(t, err) + + t.Cleanup(func() { + err := client.ExternalVolumes.Drop(ctx, sdk.NewDropExternalVolumeRequest(id)) + require.NoError(t, err) + }) + + return id + } + + t.Run("Create - S3 Storage Location", func(t *testing.T) { + allowWrites := true + comment := "some comment" + id := createExternalVolume(t, s3StorageLocations, allowWrites, comment) + + externalVolume, err := client.ExternalVolumes.ShowByID(ctx, id) + require.NoError(t, err) + + assertExternalVolumeShowResult(t, externalVolume, id, allowWrites, comment) + }) + + t.Run("Create - S3 Storage Location No Encryption", func(t *testing.T) { + allowWrites := true + comment := "some comment" + id := createExternalVolume(t, s3StorageLocationsNoneEncryption, allowWrites, comment) + + externalVolume, err := client.ExternalVolumes.ShowByID(ctx, id) + require.NoError(t, err) + + assertExternalVolumeShowResult(t, externalVolume, id, allowWrites, comment) + }) + + t.Run("Create - GCS Storage Location", func(t *testing.T) { + allowWrites := true + comment := "some comment" + id := createExternalVolume(t, gcsStorageLocations, allowWrites, comment) + + externalVolume, err := client.ExternalVolumes.ShowByID(ctx, id) + require.NoError(t, err) + + assertExternalVolumeShowResult(t, externalVolume, id, allowWrites, comment) + }) + + t.Run("Create - Azure Storage Location", func(t *testing.T) { + allowWrites := true + comment := "some comment" + id := createExternalVolume(t, azureStorageLocations, allowWrites, comment) + + externalVolume, err := client.ExternalVolumes.ShowByID(ctx, id) + require.NoError(t, err) + + assertExternalVolumeShowResult(t, externalVolume, id, allowWrites, comment) + }) + + t.Run("Create - Multiple Storage Locations", func(t *testing.T) { + allowWrites := true + comment := "some comment" + id := createExternalVolume(t, append(append(s3StorageLocations, gcsStorageLocations...), azureStorageLocations...), allowWrites, comment) + + externalVolume, err := client.ExternalVolumes.ShowByID(ctx, id) + require.NoError(t, err) + + assertExternalVolumeShowResult(t, externalVolume, id, allowWrites, comment) + }) + + t.Run("Alter - remove storage location", func(t *testing.T) { + allowWrites := true + comment := "some comment" + id := createExternalVolume(t, append(s3StorageLocationsNoneEncryption, gcsStorageLocations...), allowWrites, comment) + + req := sdk.NewAlterExternalVolumeRequest(id).WithRemoveStorageLocation(gcsStorageLocations[0].GCSStorageLocationParams.Name) + + err := client.ExternalVolumes.Alter(ctx, req) + require.NoError(t, err) + + props, err := client.ExternalVolumes.Describe(ctx, id) + require.NoError(t, err) + + trimmedProperties := trimProperties(t, props) + assert.Equal(t, 4, len(trimmedProperties)) + assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "COMMENT", Value: comment}) + assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "ALLOW_WRITES", Value: strconv.FormatBool(allowWrites)}) + assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "ACTIVE", Value: ""}) + assert.Contains( + t, + trimmedProperties, + ExternalVolumePropNameValue{ + Name: "STORAGE_LOCATION_1", + Value: fmt.Sprintf( + `{"NAME":"%s","STORAGE_PROVIDER":"%s","STORAGE_BASE_URL":"%s","STORAGE_AWS_ROLE_ARN":"%s","STORAGE_AWS_EXTERNAL_ID":"%s","ENCRYPTION_TYPE":"%s"}`, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.Name, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageProvider, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageBaseUrl, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageAwsRoleArn, + *s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageAwsExternalId, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.Encryption.Type, + ), + }, + ) + }) + + t.Run("Alter - set comment", func(t *testing.T) { + allowWrites := true + comment := "" + id := createExternalVolume(t, s3StorageLocationsNoneEncryption, allowWrites, "some comment") + + req := sdk.NewAlterExternalVolumeRequest(id).WithSet( + *sdk.NewAlterExternalVolumeSetRequest().WithComment(comment), + ) + + err := client.ExternalVolumes.Alter(ctx, req) + require.NoError(t, err) + + props, err := client.ExternalVolumes.Describe(ctx, id) + require.NoError(t, err) + + trimmedProperties := trimProperties(t, props) + assert.Equal(t, 3, len(trimmedProperties)) + assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "ALLOW_WRITES", Value: strconv.FormatBool(allowWrites)}) + assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "ACTIVE", Value: ""}) + assert.Contains( + t, + trimmedProperties, + ExternalVolumePropNameValue{ + Name: "STORAGE_LOCATION_1", + Value: fmt.Sprintf( + `{"NAME":"%s","STORAGE_PROVIDER":"%s","STORAGE_BASE_URL":"%s","STORAGE_AWS_ROLE_ARN":"%s","STORAGE_AWS_EXTERNAL_ID":"%s","ENCRYPTION_TYPE":"%s"}`, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.Name, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageProvider, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageBaseUrl, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageAwsRoleArn, + *s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageAwsExternalId, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.Encryption.Type, + ), + }, + ) + }) + + t.Run("Alter - set allow writes", func(t *testing.T) { + allowWrites := false + comment := "some comment" + id := createExternalVolume(t, s3StorageLocations, true, comment) + + req := sdk.NewAlterExternalVolumeRequest(id).WithSet( + *sdk.NewAlterExternalVolumeSetRequest().WithAllowWrites(allowWrites), + ) + + err := client.ExternalVolumes.Alter(ctx, req) + require.NoError(t, err) + + props, err := client.ExternalVolumes.Describe(ctx, id) + require.NoError(t, err) + + trimmedProperties := trimProperties(t, props) + assert.Equal(t, 4, len(trimmedProperties)) + assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "COMMENT", Value: comment}) + assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "ALLOW_WRITES", Value: strconv.FormatBool(allowWrites)}) + assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "ACTIVE", Value: ""}) + assert.Contains( + t, + trimmedProperties, + ExternalVolumePropNameValue{ + Name: "STORAGE_LOCATION_1", + Value: fmt.Sprintf( + `{"NAME":"%s","STORAGE_PROVIDER":"%s","STORAGE_BASE_URL":"%s","STORAGE_AWS_ROLE_ARN":"%s","STORAGE_AWS_EXTERNAL_ID":"%s","ENCRYPTION_TYPE":"%s","ENCRYPTION_KMS_KEY_ID":"%s"}`, + s3StorageLocations[0].S3StorageLocationParams.Name, + s3StorageLocations[0].S3StorageLocationParams.StorageProvider, + s3StorageLocations[0].S3StorageLocationParams.StorageBaseUrl, + s3StorageLocations[0].S3StorageLocationParams.StorageAwsRoleArn, + *s3StorageLocations[0].S3StorageLocationParams.StorageAwsExternalId, + s3StorageLocations[0].S3StorageLocationParams.Encryption.Type, + *s3StorageLocations[0].S3StorageLocationParams.Encryption.KmsKeyId, + ), + }, + ) + }) + + t.Run("Alter - add s3 storage location to external volume", func(t *testing.T) { + allowWrites := true + comment := "some comment" + id := createExternalVolume(t, gcsStorageLocations, allowWrites, comment) + + req := sdk.NewAlterExternalVolumeRequest(id).WithAddStorageLocation( + *sdk.NewExternalVolumeStorageLocationRequest().WithS3StorageLocationParams( + *sdk.NewS3StorageLocationParamsRequest( + s3StorageLocations[0].S3StorageLocationParams.Name, + s3StorageLocations[0].S3StorageLocationParams.StorageProvider, + s3StorageLocations[0].S3StorageLocationParams.StorageAwsRoleArn, + s3StorageLocations[0].S3StorageLocationParams.StorageBaseUrl, + *sdk.NewExternalVolumeS3EncryptionRequest( + s3StorageLocations[0].S3StorageLocationParams.Encryption.Type, + ).WithKmsKeyId(*s3StorageLocations[0].S3StorageLocationParams.Encryption.KmsKeyId,), + ).WithStorageAwsExternalId(*s3StorageLocations[0].S3StorageLocationParams.StorageAwsExternalId,), + ), + ) + + err := client.ExternalVolumes.Alter(ctx, req) + require.NoError(t, err) + + props, err := client.ExternalVolumes.Describe(ctx, id) + require.NoError(t, err) + + trimmedProperties := trimProperties(t, props) + assert.Equal(t, 5, len(trimmedProperties)) + assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "COMMENT", Value: comment}) + assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "ALLOW_WRITES", Value: strconv.FormatBool(allowWrites)}) + assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "ACTIVE", Value: ""}) + assert.Contains( + t, + trimmedProperties, + ExternalVolumePropNameValue{ + Name: "STORAGE_LOCATION_1", + Value: fmt.Sprintf( + `{"NAME":"%s","STORAGE_PROVIDER":"GCS","STORAGE_BASE_URL":"%s","ENCRYPTION_TYPE":"%s"}`, + gcsStorageLocations[0].GCSStorageLocationParams.Name, + gcsStorageLocations[0].GCSStorageLocationParams.StorageBaseUrl, + gcsStorageLocations[0].GCSStorageLocationParams.Encryption.Type, + ), + }, + ) + assert.Contains( + t, + trimmedProperties, + ExternalVolumePropNameValue{ + Name: "STORAGE_LOCATION_2", + Value: fmt.Sprintf( + `{"NAME":"%s","STORAGE_PROVIDER":"%s","STORAGE_BASE_URL":"%s","STORAGE_AWS_ROLE_ARN":"%s","STORAGE_AWS_EXTERNAL_ID":"%s","ENCRYPTION_TYPE":"%s","ENCRYPTION_KMS_KEY_ID":"%s"}`, + s3StorageLocations[0].S3StorageLocationParams.Name, + s3StorageLocations[0].S3StorageLocationParams.StorageProvider, + s3StorageLocations[0].S3StorageLocationParams.StorageBaseUrl, + s3StorageLocations[0].S3StorageLocationParams.StorageAwsRoleArn, + *s3StorageLocations[0].S3StorageLocationParams.StorageAwsExternalId, + s3StorageLocations[0].S3StorageLocationParams.Encryption.Type, + *s3StorageLocations[0].S3StorageLocationParams.Encryption.KmsKeyId, + ), + }, + ) + }) + + t.Run("Describe", func(t *testing.T) { + allowWrites := true + comment := "some comment" + id := createExternalVolume( + t, + append(append(append(s3StorageLocations, gcsStorageLocations...), azureStorageLocations...), s3StorageLocationsNoneEncryption...), + allowWrites, + comment, + ) + + props, err := client.ExternalVolumes.Describe(ctx, id) + require.NoError(t, err) + + trimmedProperties := trimProperties(t, props) + assert.Equal(t, 7, len(trimmedProperties)) + assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "COMMENT", Value: comment}) + assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "ALLOW_WRITES", Value: strconv.FormatBool(allowWrites)}) + assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "ACTIVE", Value: ""}) + assert.Contains( + t, + trimmedProperties, + ExternalVolumePropNameValue{ + Name: "STORAGE_LOCATION_1", + Value: fmt.Sprintf( + `{"NAME":"%s","STORAGE_PROVIDER":"%s","STORAGE_BASE_URL":"%s","STORAGE_AWS_ROLE_ARN":"%s","STORAGE_AWS_EXTERNAL_ID":"%s","ENCRYPTION_TYPE":"%s","ENCRYPTION_KMS_KEY_ID":"%s"}`, + s3StorageLocations[0].S3StorageLocationParams.Name, + s3StorageLocations[0].S3StorageLocationParams.StorageProvider, + s3StorageLocations[0].S3StorageLocationParams.StorageBaseUrl, + s3StorageLocations[0].S3StorageLocationParams.StorageAwsRoleArn, + *s3StorageLocations[0].S3StorageLocationParams.StorageAwsExternalId, + s3StorageLocations[0].S3StorageLocationParams.Encryption.Type, + *s3StorageLocations[0].S3StorageLocationParams.Encryption.KmsKeyId, + ), + }, + ) + assert.Contains( + t, + trimmedProperties, + ExternalVolumePropNameValue{ + Name: "STORAGE_LOCATION_2", + Value: fmt.Sprintf( + `{"NAME":"%s","STORAGE_PROVIDER":"GCS","STORAGE_BASE_URL":"%s","ENCRYPTION_TYPE":"%s"}`, + gcsStorageLocations[0].GCSStorageLocationParams.Name, + gcsStorageLocations[0].GCSStorageLocationParams.StorageBaseUrl, + gcsStorageLocations[0].GCSStorageLocationParams.Encryption.Type, + ), + }, + ) + assert.Contains( + t, + trimmedProperties, + ExternalVolumePropNameValue{ + Name: "STORAGE_LOCATION_3", + Value: fmt.Sprintf( + `{"NAME":"%s","STORAGE_PROVIDER":"AZURE","STORAGE_BASE_URL":"%s","AZURE_TENANT_ID":"%s"}`, + azureStorageLocations[0].AzureStorageLocationParams.Name, + azureStorageLocations[0].AzureStorageLocationParams.StorageBaseUrl, + azureStorageLocations[0].AzureStorageLocationParams.AzureTenantId, + ), + }, + ) + assert.Contains( + t, + trimmedProperties, + ExternalVolumePropNameValue{ + Name: "STORAGE_LOCATION_4", + Value: fmt.Sprintf( + `{"NAME":"%s","STORAGE_PROVIDER":"%s","STORAGE_BASE_URL":"%s","STORAGE_AWS_ROLE_ARN":"%s","STORAGE_AWS_EXTERNAL_ID":"%s","ENCRYPTION_TYPE":"%s"}`, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.Name, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageProvider, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageBaseUrl, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageAwsRoleArn, + *s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageAwsExternalId, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.Encryption.Type, + ), + }, + ) + }) + + t.Run("Show with like", func(t *testing.T) { + allowWrites := true + comment := "some comment" + id := createExternalVolume(t, s3StorageLocations, allowWrites, comment) + name := id.Name() + req := sdk.NewShowExternalVolumeRequest().WithLike(sdk.Like{Pattern: &name}) + + externalVolumes, err := client.ExternalVolumes.Show(ctx, req) + require.NoError(t, err) + + assert.Equal(t, 1, len(externalVolumes)) + assertExternalVolumeShowResult(t, &externalVolumes[0], id, allowWrites, comment) + }) +} From aca87364495d862e364ec3f8cf23e01932c1bcd8 Mon Sep 17 00:00:00 2001 From: James Doldissen Date: Mon, 2 Sep 2024 23:22:31 +0200 Subject: [PATCH 02/10] fix: Run linter --- pkg/sdk/external_volumes_def.go | 170 ++-- pkg/sdk/external_volumes_gen_test.go | 255 +++--- pkg/sdk/external_volumes_impl_gen.go | 15 +- pkg/sdk/external_volumes_validations_gen.go | 22 +- .../external_volumes_gen_integration_test.go | 799 +++++++++--------- 5 files changed, 628 insertions(+), 633 deletions(-) diff --git a/pkg/sdk/external_volumes_def.go b/pkg/sdk/external_volumes_def.go index 5c43d62361..01c7d00174 100644 --- a/pkg/sdk/external_volumes_def.go +++ b/pkg/sdk/external_volumes_def.go @@ -4,69 +4,71 @@ import g "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/sdk/poc/gen //go:generate go run ./poc/main.go -type S3EncryptionType string -type S3StorageProvider string -type GCSEncryptionType string +type ( + S3EncryptionType string + S3StorageProvider string + GCSEncryptionType string +) var ( S3EncryptionTypeSseS3 S3EncryptionType = "AWS_SSE_S3" S3EncryptionTypeSseKms S3EncryptionType = "AWS_SSE_KMS" S3EncryptionNone S3EncryptionType = "NONE" - GCSEncryptionTypeSseKms GCSEncryptionType = "GCS_SSE_KMS" - GCSEncryptionTypeNone GCSEncryptionType = "NONE" - S3StorageProviderS3 S3StorageProvider = "S3" - S3StorageProviderS3GOV S3StorageProvider = "S3GOV" + GCSEncryptionTypeSseKms GCSEncryptionType = "GCS_SSE_KMS" + GCSEncryptionTypeNone GCSEncryptionType = "NONE" + S3StorageProviderS3 S3StorageProvider = "S3" + S3StorageProviderS3GOV S3StorageProvider = "S3GOV" ) var externalS3StorageLocationDef = g.NewQueryStruct("S3StorageLocationParams"). - TextAssignment("NAME", g.ParameterOptions().SingleQuotes().Required()). - Assignment("STORAGE_PROVIDER", g.KindOfT[S3StorageProvider](), g.ParameterOptions().SingleQuotes().Required()). - TextAssignment("STORAGE_AWS_ROLE_ARN", g.ParameterOptions().SingleQuotes().Required()). - TextAssignment("STORAGE_BASE_URL", g.ParameterOptions().SingleQuotes().Required()). - OptionalTextAssignment("STORAGE_AWS_EXTERNAL_ID", g.ParameterOptions().SingleQuotes()). - QueryStructField( - "Encryption", - g.NewQueryStruct("ExternalVolumeS3Encryption"). - Assignment("TYPE", g.KindOfT[S3EncryptionType](), g.ParameterOptions().SingleQuotes().Required()). - OptionalTextAssignment("KMS_KEY_ID", g.ParameterOptions().SingleQuotes()), - g.ListOptions().Parentheses().NoComma().SQL("ENCRYPTION =").Required(), - ) + TextAssignment("NAME", g.ParameterOptions().SingleQuotes().Required()). + Assignment("STORAGE_PROVIDER", g.KindOfT[S3StorageProvider](), g.ParameterOptions().SingleQuotes().Required()). + TextAssignment("STORAGE_AWS_ROLE_ARN", g.ParameterOptions().SingleQuotes().Required()). + TextAssignment("STORAGE_BASE_URL", g.ParameterOptions().SingleQuotes().Required()). + OptionalTextAssignment("STORAGE_AWS_EXTERNAL_ID", g.ParameterOptions().SingleQuotes()). + QueryStructField( + "Encryption", + g.NewQueryStruct("ExternalVolumeS3Encryption"). + Assignment("TYPE", g.KindOfT[S3EncryptionType](), g.ParameterOptions().SingleQuotes().Required()). + OptionalTextAssignment("KMS_KEY_ID", g.ParameterOptions().SingleQuotes()), + g.ListOptions().Parentheses().NoComma().SQL("ENCRYPTION =").Required(), + ) var externalGCSStorageLocationDef = g.NewQueryStruct("GCSStorageLocationParams"). - TextAssignment("NAME", g.ParameterOptions().SingleQuotes().Required()). - PredefinedQueryStructField("StorageProviderGcs", "string", g.StaticOptions().SQL("STORAGE_PROVIDER = 'GCS'")). - TextAssignment("STORAGE_BASE_URL", g.ParameterOptions().SingleQuotes().Required()). - QueryStructField( - "Encryption", - g.NewQueryStruct("ExternalVolumeGCSEncryption"). - Assignment("TYPE", g.KindOfT[GCSEncryptionType](), g.ParameterOptions().SingleQuotes().Required()). - OptionalTextAssignment("KMS_KEY_ID", g.ParameterOptions().SingleQuotes()), - g.ListOptions().Parentheses().NoComma().SQL("ENCRYPTION =").Required(), - ) + TextAssignment("NAME", g.ParameterOptions().SingleQuotes().Required()). + PredefinedQueryStructField("StorageProviderGcs", "string", g.StaticOptions().SQL("STORAGE_PROVIDER = 'GCS'")). + TextAssignment("STORAGE_BASE_URL", g.ParameterOptions().SingleQuotes().Required()). + QueryStructField( + "Encryption", + g.NewQueryStruct("ExternalVolumeGCSEncryption"). + Assignment("TYPE", g.KindOfT[GCSEncryptionType](), g.ParameterOptions().SingleQuotes().Required()). + OptionalTextAssignment("KMS_KEY_ID", g.ParameterOptions().SingleQuotes()), + g.ListOptions().Parentheses().NoComma().SQL("ENCRYPTION =").Required(), + ) var externalAzureStorageLocationDef = g.NewQueryStruct("AzureStorageLocationParams"). - TextAssignment("NAME", g.ParameterOptions().SingleQuotes().Required()). - PredefinedQueryStructField("StorageProviderAzure", "string", g.StaticOptions().SQL("STORAGE_PROVIDER = 'AZURE'")). - TextAssignment("AZURE_TENANT_ID", g.ParameterOptions().SingleQuotes().Required()). - TextAssignment("STORAGE_BASE_URL", g.ParameterOptions().SingleQuotes().Required()) + TextAssignment("NAME", g.ParameterOptions().SingleQuotes().Required()). + PredefinedQueryStructField("StorageProviderAzure", "string", g.StaticOptions().SQL("STORAGE_PROVIDER = 'AZURE'")). + TextAssignment("AZURE_TENANT_ID", g.ParameterOptions().SingleQuotes().Required()). + TextAssignment("STORAGE_BASE_URL", g.ParameterOptions().SingleQuotes().Required()) var storageLocationDef = g.NewQueryStruct("ExternalVolumeStorageLocation"). // Can't name StorageLocation due to naming clash with type in storage integration - OptionalQueryStructField( - "S3StorageLocationParams", - externalS3StorageLocationDef, - g.ListOptions().Parentheses().NoComma(), - ). - OptionalQueryStructField( - "GCSStorageLocationParams", - externalGCSStorageLocationDef, - g.ListOptions().Parentheses().NoComma(), - ). - OptionalQueryStructField( - "AzureStorageLocationParams", - externalAzureStorageLocationDef, - g.ListOptions().Parentheses().NoComma(), - ). - WithValidation(g.ExactlyOneValueSet, "S3StorageLocationParams", "GCSStorageLocationParams", "AzureStorageLocationParams") + OptionalQueryStructField( + "S3StorageLocationParams", + externalS3StorageLocationDef, + g.ListOptions().Parentheses().NoComma(), + ). + OptionalQueryStructField( + "GCSStorageLocationParams", + externalGCSStorageLocationDef, + g.ListOptions().Parentheses().NoComma(), + ). + OptionalQueryStructField( + "AzureStorageLocationParams", + externalAzureStorageLocationDef, + g.ListOptions().Parentheses().NoComma(), + ). + WithValidation(g.ExactlyOneValueSet, "S3StorageLocationParams", "GCSStorageLocationParams", "AzureStorageLocationParams") var ExternalVolumesDef = g.NewInterface( "ExternalVolumes", @@ -75,42 +77,42 @@ var ExternalVolumesDef = g.NewInterface( ). CreateOperation( "https://docs.snowflake.com/en/sql-reference/sql/create-external-volume", - g.NewQueryStruct("CreateExternalVolume"). - Create(). - OrReplace(). - SQL("EXTERNAL VOLUME"). - IfNotExists(). - Name(). - ListAssignment("STORAGE_LOCATIONS", "ExternalVolumeStorageLocation", g.ParameterOptions().Parentheses().Required()). - OptionalBooleanAssignment("ALLOW_WRITES", nil). - OptionalComment(). - WithValidation(g.ConflictingFields, "OrReplace", "IfNotExists"). - WithValidation(g.ValidIdentifier, "name"), - storageLocationDef, + g.NewQueryStruct("CreateExternalVolume"). + Create(). + OrReplace(). + SQL("EXTERNAL VOLUME"). + IfNotExists(). + Name(). + ListAssignment("STORAGE_LOCATIONS", "ExternalVolumeStorageLocation", g.ParameterOptions().Parentheses().Required()). + OptionalBooleanAssignment("ALLOW_WRITES", nil). + OptionalComment(). + WithValidation(g.ConflictingFields, "OrReplace", "IfNotExists"). + WithValidation(g.ValidIdentifier, "name"), + storageLocationDef, + ). + AlterOperation( + "https://docs.snowflake.com/en/sql-reference/sql/alter-external-volume", + g.NewQueryStruct("AlterExternalVolume"). + Alter(). + SQL("EXTERNAL VOLUME"). + IfExists(). + Name(). + OptionalTextAssignment("REMOVE STORAGE_LOCATION", g.ParameterOptions().SingleQuotes().NoEquals()). + OptionalQueryStructField( + "Set", + g.NewQueryStruct("AlterExternalVolumeSet"). + OptionalBooleanAssignment("ALLOW_WRITES", g.ParameterOptions()). + OptionalComment(), + g.KeywordOptions().SQL("SET"), + ). + OptionalQueryStructField( + "AddStorageLocation", + storageLocationDef, + g.ParameterOptions().SQL("ADD STORAGE_LOCATION"), + ). + WithValidation(g.ExactlyOneValueSet, "RemoveStorageLocation", "Set", "AddStorageLocation"). + WithValidation(g.ValidIdentifier, "name"), ). - AlterOperation( - "https://docs.snowflake.com/en/sql-reference/sql/alter-external-volume", - g.NewQueryStruct("AlterExternalVolume"). - Alter(). - SQL("EXTERNAL VOLUME"). - IfExists(). - Name(). - OptionalTextAssignment("REMOVE STORAGE_LOCATION", g.ParameterOptions().SingleQuotes().NoEquals()). - OptionalQueryStructField( - "Set", - g.NewQueryStruct("AlterExternalVolumeSet"). - OptionalBooleanAssignment("ALLOW_WRITES", g.ParameterOptions()). - OptionalComment(), - g.KeywordOptions().SQL("SET"), - ). - OptionalQueryStructField( - "AddStorageLocation", - storageLocationDef, - g.ParameterOptions().SQL("ADD STORAGE_LOCATION"), - ). - WithValidation(g.ExactlyOneValueSet, "RemoveStorageLocation", "Set", "AddStorageLocation"). - WithValidation(g.ValidIdentifier, "name"), - ). DropOperation( "https://docs.snowflake.com/en/sql-reference/sql/drop-external-volume", g.NewQueryStruct("DropExternalVolume"). diff --git a/pkg/sdk/external_volumes_gen_test.go b/pkg/sdk/external_volumes_gen_test.go index 3edc069b80..15c3118ceb 100644 --- a/pkg/sdk/external_volumes_gen_test.go +++ b/pkg/sdk/external_volumes_gen_test.go @@ -4,75 +4,73 @@ import "testing" // Storage location structs for testing var s3StorageLocationParams = &S3StorageLocationParams{ - Name: "some s3 name", - StorageProvider: S3StorageProviderS3, - StorageAwsRoleArn: "some s3 role arn", - StorageBaseUrl: "some s3 base url", - Encryption: ExternalVolumeS3Encryption{ - Type: S3EncryptionTypeSseS3, - KmsKeyId: String("some s3 kms key id"), - }, + Name: "some s3 name", + StorageProvider: S3StorageProviderS3, + StorageAwsRoleArn: "some s3 role arn", + StorageBaseUrl: "some s3 base url", + Encryption: ExternalVolumeS3Encryption{ + Type: S3EncryptionTypeSseS3, + KmsKeyId: String("some s3 kms key id"), + }, } var s3StorageLocationParamsNoEncryption = &S3StorageLocationParams{ - Name: "some s3 name", - StorageProvider: S3StorageProviderS3, - StorageAwsRoleArn: "some s3 role arn", - StorageBaseUrl: "some s3 base url", - Encryption: ExternalVolumeS3Encryption{ - Type: S3EncryptionNone, - }, + Name: "some s3 name", + StorageProvider: S3StorageProviderS3, + StorageAwsRoleArn: "some s3 role arn", + StorageBaseUrl: "some s3 base url", + Encryption: ExternalVolumeS3Encryption{ + Type: S3EncryptionNone, + }, } var s3StorageLocationParamsGov = &S3StorageLocationParams{ - Name: "some s3 name", - StorageProvider: S3StorageProviderS3GOV, - StorageAwsRoleArn: "some s3 role arn", - StorageBaseUrl: "some s3 base url", - Encryption: ExternalVolumeS3Encryption{ - Type: S3EncryptionTypeSseS3, - KmsKeyId: String("some s3 kms key id"), - }, + Name: "some s3 name", + StorageProvider: S3StorageProviderS3GOV, + StorageAwsRoleArn: "some s3 role arn", + StorageBaseUrl: "some s3 base url", + Encryption: ExternalVolumeS3Encryption{ + Type: S3EncryptionTypeSseS3, + KmsKeyId: String("some s3 kms key id"), + }, } var s3StorageLocationParamsWithExternalId = &S3StorageLocationParams{ - Name: "some s3 name", - StorageProvider: S3StorageProviderS3, - StorageAwsRoleArn: "some s3 role arn", - StorageBaseUrl: "some s3 base url", - StorageAwsExternalId: String("some s3 external id"), - Encryption: ExternalVolumeS3Encryption{ - Type: S3EncryptionTypeSseS3, - KmsKeyId: String("some s3 kms key id"), - }, + Name: "some s3 name", + StorageProvider: S3StorageProviderS3, + StorageAwsRoleArn: "some s3 role arn", + StorageBaseUrl: "some s3 base url", + StorageAwsExternalId: String("some s3 external id"), + Encryption: ExternalVolumeS3Encryption{ + Type: S3EncryptionTypeSseS3, + KmsKeyId: String("some s3 kms key id"), + }, } var gcsStorageLocationParams = &GCSStorageLocationParams{ - Name: "some gcs name", - StorageBaseUrl: "some gcs base url", - Encryption: ExternalVolumeGCSEncryption{ - Type: GCSEncryptionTypeSseKms, - KmsKeyId: String("some gcs kms key id"), - }, + Name: "some gcs name", + StorageBaseUrl: "some gcs base url", + Encryption: ExternalVolumeGCSEncryption{ + Type: GCSEncryptionTypeSseKms, + KmsKeyId: String("some gcs kms key id"), + }, } var gcsStorageLocationParamsNoEncryption = &GCSStorageLocationParams{ - Name: "some gcs name", - StorageBaseUrl: "some gcs base url", - Encryption: ExternalVolumeGCSEncryption{ - Type: GCSEncryptionTypeNone, - }, + Name: "some gcs name", + StorageBaseUrl: "some gcs base url", + Encryption: ExternalVolumeGCSEncryption{ + Type: GCSEncryptionTypeNone, + }, } var azureStorageLocationParams = &AzureStorageLocationParams{ - Name: "some azure name", - AzureTenantId: "some azure tenant id", - StorageBaseUrl: "some azure base url", + Name: "some azure name", + AzureTenantId: "some azure tenant id", + StorageBaseUrl: "some azure base url", } - func TestExternalVolumes_Create(t *testing.T) { - id := randomAccountObjectIdentifier() // Minimal valid CreateExternalVolumeOptions defaultOpts := func() *CreateExternalVolumeOptions { @@ -95,33 +93,33 @@ func TestExternalVolumes_Create(t *testing.T) { t.Run("validation: valid identifier for [opts.name]", func(t *testing.T) { opts := defaultOpts() - opts.name = emptyAccountObjectIdentifier + opts.name = emptyAccountObjectIdentifier assertOptsInvalidJoinedErrors(t, opts, ErrInvalidObjectIdentifier) }) t.Run("validation: exactly one field from [opts.StorageLocations[i].S3StorageLocationParams opts.StorageLocations[i].GCSStorageLocationParams opts.StorageLocations[i].AzureStorageLocationParams] should be present", func(t *testing.T) { opts := defaultOpts() opts.StorageLocations = []ExternalVolumeStorageLocation{ - { S3StorageLocationParams: s3StorageLocationParams, }, - {}, - { S3StorageLocationParams: s3StorageLocationParams, GCSStorageLocationParams: gcsStorageLocationParams, }, - { S3StorageLocationParams: s3StorageLocationParams, AzureStorageLocationParams: azureStorageLocationParams, }, - { GCSStorageLocationParams: gcsStorageLocationParams, AzureStorageLocationParams: azureStorageLocationParams, }, - { - S3StorageLocationParams: s3StorageLocationParams, - GCSStorageLocationParams: gcsStorageLocationParams, - AzureStorageLocationParams: azureStorageLocationParams, - }, + {S3StorageLocationParams: s3StorageLocationParams}, + {}, + {S3StorageLocationParams: s3StorageLocationParams, GCSStorageLocationParams: gcsStorageLocationParams}, + {S3StorageLocationParams: s3StorageLocationParams, AzureStorageLocationParams: azureStorageLocationParams}, + {GCSStorageLocationParams: gcsStorageLocationParams, AzureStorageLocationParams: azureStorageLocationParams}, + { + S3StorageLocationParams: s3StorageLocationParams, + GCSStorageLocationParams: gcsStorageLocationParams, + AzureStorageLocationParams: azureStorageLocationParams, + }, } assertOptsInvalidJoinedErrors( - t, - opts, - errExactlyOneOf("CreateExternalVolumeOptions.StorageLocation[1]", "S3StorageLocationParams", "GCSStorageLocationParams", "AzureStorageLocationParams"), - errExactlyOneOf("CreateExternalVolumeOptions.StorageLocation[2]", "S3StorageLocationParams", "GCSStorageLocationParams", "AzureStorageLocationParams"), - errExactlyOneOf("CreateExternalVolumeOptions.StorageLocation[3]", "S3StorageLocationParams", "GCSStorageLocationParams", "AzureStorageLocationParams"), - errExactlyOneOf("CreateExternalVolumeOptions.StorageLocation[4]", "S3StorageLocationParams", "GCSStorageLocationParams", "AzureStorageLocationParams"), - errExactlyOneOf("CreateExternalVolumeOptions.StorageLocation[5]", "S3StorageLocationParams", "GCSStorageLocationParams", "AzureStorageLocationParams"), - ) + t, + opts, + errExactlyOneOf("CreateExternalVolumeOptions.StorageLocation[1]", "S3StorageLocationParams", "GCSStorageLocationParams", "AzureStorageLocationParams"), + errExactlyOneOf("CreateExternalVolumeOptions.StorageLocation[2]", "S3StorageLocationParams", "GCSStorageLocationParams", "AzureStorageLocationParams"), + errExactlyOneOf("CreateExternalVolumeOptions.StorageLocation[3]", "S3StorageLocationParams", "GCSStorageLocationParams", "AzureStorageLocationParams"), + errExactlyOneOf("CreateExternalVolumeOptions.StorageLocation[4]", "S3StorageLocationParams", "GCSStorageLocationParams", "AzureStorageLocationParams"), + errExactlyOneOf("CreateExternalVolumeOptions.StorageLocation[5]", "S3StorageLocationParams", "GCSStorageLocationParams", "AzureStorageLocationParams"), + ) }) t.Run("validation: length of opts.StorageLocations is > 0", func(t *testing.T) { @@ -132,59 +130,58 @@ func TestExternalVolumes_Create(t *testing.T) { t.Run("1 storage location - s3", func(t *testing.T) { opts := defaultOpts() - opts.StorageLocations = []ExternalVolumeStorageLocation{ { S3StorageLocationParams: s3StorageLocationParams, }, } + opts.StorageLocations = []ExternalVolumeStorageLocation{{S3StorageLocationParams: s3StorageLocationParams}} assertOptsValidAndSQLEquals(t, opts, `CREATE EXTERNAL VOLUME %s STORAGE_LOCATIONS = ((NAME = 'some s3 name' STORAGE_PROVIDER = 'S3' STORAGE_AWS_ROLE_ARN = 'some s3 role arn' STORAGE_BASE_URL = 'some s3 base url' ENCRYPTION = (TYPE = 'AWS_SSE_S3' KMS_KEY_ID = 'some s3 kms key id')))`, id.FullyQualifiedName()) }) t.Run("1 storage location with comment - s3gov", func(t *testing.T) { opts := defaultOpts() - opts.StorageLocations = []ExternalVolumeStorageLocation{ { S3StorageLocationParams: s3StorageLocationParamsGov, }, } - opts.Comment = String("some comment") + opts.StorageLocations = []ExternalVolumeStorageLocation{{S3StorageLocationParams: s3StorageLocationParamsGov}} + opts.Comment = String("some comment") assertOptsValidAndSQLEquals(t, opts, `CREATE EXTERNAL VOLUME %s STORAGE_LOCATIONS = ((NAME = 'some s3 name' STORAGE_PROVIDER = 'S3GOV' STORAGE_AWS_ROLE_ARN = 'some s3 role arn' STORAGE_BASE_URL = 'some s3 base url' ENCRYPTION = (TYPE = 'AWS_SSE_S3' KMS_KEY_ID = 'some s3 kms key id'))) COMMENT = 'some comment'`, id.FullyQualifiedName()) }) t.Run("1 storage location - s3 no encryption", func(t *testing.T) { opts := defaultOpts() - opts.StorageLocations = []ExternalVolumeStorageLocation{ { S3StorageLocationParams: s3StorageLocationParamsNoEncryption, }, } + opts.StorageLocations = []ExternalVolumeStorageLocation{{S3StorageLocationParams: s3StorageLocationParamsNoEncryption}} assertOptsValidAndSQLEquals(t, opts, `CREATE EXTERNAL VOLUME %s STORAGE_LOCATIONS = ((NAME = 'some s3 name' STORAGE_PROVIDER = 'S3' STORAGE_AWS_ROLE_ARN = 'some s3 role arn' STORAGE_BASE_URL = 'some s3 base url' ENCRYPTION = (TYPE = 'NONE')))`, id.FullyQualifiedName()) }) t.Run("1 storage location with allow writes - gcs", func(t *testing.T) { opts := defaultOpts() - opts.StorageLocations = []ExternalVolumeStorageLocation{ { GCSStorageLocationParams: gcsStorageLocationParams, }, } - opts.AllowWrites = Bool(true) + opts.StorageLocations = []ExternalVolumeStorageLocation{{GCSStorageLocationParams: gcsStorageLocationParams}} + opts.AllowWrites = Bool(true) assertOptsValidAndSQLEquals(t, opts, `CREATE EXTERNAL VOLUME %s STORAGE_LOCATIONS = ((NAME = 'some gcs name' STORAGE_PROVIDER = 'GCS' STORAGE_BASE_URL = 'some gcs base url' ENCRYPTION = (TYPE = 'GCS_SSE_KMS' KMS_KEY_ID = 'some gcs kms key id'))) ALLOW_WRITES = true`, id.FullyQualifiedName()) }) t.Run("1 storage location with allow writes - gcs no encryption", func(t *testing.T) { opts := defaultOpts() - opts.StorageLocations = []ExternalVolumeStorageLocation{ { GCSStorageLocationParams: gcsStorageLocationParamsNoEncryption, }, } - opts.AllowWrites = Bool(true) + opts.StorageLocations = []ExternalVolumeStorageLocation{{GCSStorageLocationParams: gcsStorageLocationParamsNoEncryption}} + opts.AllowWrites = Bool(true) assertOptsValidAndSQLEquals(t, opts, `CREATE EXTERNAL VOLUME %s STORAGE_LOCATIONS = ((NAME = 'some gcs name' STORAGE_PROVIDER = 'GCS' STORAGE_BASE_URL = 'some gcs base url' ENCRYPTION = (TYPE = 'NONE'))) ALLOW_WRITES = true`, id.FullyQualifiedName()) }) t.Run("1 storage location - azure", func(t *testing.T) { opts := defaultOpts() - opts.StorageLocations = []ExternalVolumeStorageLocation{ { AzureStorageLocationParams: azureStorageLocationParams, }, } + opts.StorageLocations = []ExternalVolumeStorageLocation{{AzureStorageLocationParams: azureStorageLocationParams}} assertOptsValidAndSQLEquals(t, opts, `CREATE EXTERNAL VOLUME %s STORAGE_LOCATIONS = ((NAME = 'some azure name' STORAGE_PROVIDER = 'AZURE' AZURE_TENANT_ID = 'some azure tenant id' STORAGE_BASE_URL = 'some azure base url'))`, id.FullyQualifiedName()) }) - t.Run("3 storage locations and all options", func(t *testing.T) { - opts := defaultOpts() - opts.OrReplace = Bool(true) - opts.StorageLocations = []ExternalVolumeStorageLocation{ - { S3StorageLocationParams: s3StorageLocationParamsWithExternalId, }, - { GCSStorageLocationParams: gcsStorageLocationParams, }, - { AzureStorageLocationParams: azureStorageLocationParams, }, - } - opts.AllowWrites = Bool(false) - opts.Comment = String("some comment") - assertOptsValidAndSQLEquals(t, opts, `CREATE OR REPLACE EXTERNAL VOLUME %s STORAGE_LOCATIONS = ((NAME = 'some s3 name' STORAGE_PROVIDER = 'S3' STORAGE_AWS_ROLE_ARN = 'some s3 role arn' STORAGE_BASE_URL = 'some s3 base url' STORAGE_AWS_EXTERNAL_ID = 'some s3 external id' ENCRYPTION = (TYPE = 'AWS_SSE_S3' KMS_KEY_ID = 'some s3 kms key id')), (NAME = 'some gcs name' STORAGE_PROVIDER = 'GCS' STORAGE_BASE_URL = 'some gcs base url' ENCRYPTION = (TYPE = 'GCS_SSE_KMS' KMS_KEY_ID = 'some gcs kms key id')), (NAME = 'some azure name' STORAGE_PROVIDER = 'AZURE' AZURE_TENANT_ID = 'some azure tenant id' STORAGE_BASE_URL = 'some azure base url')) ALLOW_WRITES = false COMMENT = 'some comment'`, id.FullyQualifiedName()) - }) + t.Run("3 storage locations and all options", func(t *testing.T) { + opts := defaultOpts() + opts.OrReplace = Bool(true) + opts.StorageLocations = []ExternalVolumeStorageLocation{ + {S3StorageLocationParams: s3StorageLocationParamsWithExternalId}, + {GCSStorageLocationParams: gcsStorageLocationParams}, + {AzureStorageLocationParams: azureStorageLocationParams}, + } + opts.AllowWrites = Bool(false) + opts.Comment = String("some comment") + assertOptsValidAndSQLEquals(t, opts, `CREATE OR REPLACE EXTERNAL VOLUME %s STORAGE_LOCATIONS = ((NAME = 'some s3 name' STORAGE_PROVIDER = 'S3' STORAGE_AWS_ROLE_ARN = 'some s3 role arn' STORAGE_BASE_URL = 'some s3 base url' STORAGE_AWS_EXTERNAL_ID = 'some s3 external id' ENCRYPTION = (TYPE = 'AWS_SSE_S3' KMS_KEY_ID = 'some s3 kms key id')), (NAME = 'some gcs name' STORAGE_PROVIDER = 'GCS' STORAGE_BASE_URL = 'some gcs base url' ENCRYPTION = (TYPE = 'GCS_SSE_KMS' KMS_KEY_ID = 'some gcs kms key id')), (NAME = 'some azure name' STORAGE_PROVIDER = 'AZURE' AZURE_TENANT_ID = 'some azure tenant id' STORAGE_BASE_URL = 'some azure base url')) ALLOW_WRITES = false COMMENT = 'some comment'`, id.FullyQualifiedName()) + }) } func TestExternalVolumes_Alter(t *testing.T) { - id := randomAccountObjectIdentifier() // Minimal valid AlterExternalVolumeOptions defaultOpts := func() *AlterExternalVolumeOptions { @@ -203,14 +200,14 @@ func TestExternalVolumes_Alter(t *testing.T) { removeAndAddOpts := defaultOpts() setAndAddOpts := defaultOpts() - removeAndSetOpts.RemoveStorageLocation = String("some storage location") - removeAndSetOpts.Set = &AlterExternalVolumeSet{ AllowWrites: Bool(true), } + removeAndSetOpts.RemoveStorageLocation = String("some storage location") + removeAndSetOpts.Set = &AlterExternalVolumeSet{AllowWrites: Bool(true)} - removeAndAddOpts.RemoveStorageLocation = String("some storage location") - removeAndAddOpts.AddStorageLocation = &ExternalVolumeStorageLocation{ S3StorageLocationParams: s3StorageLocationParamsWithExternalId, } + removeAndAddOpts.RemoveStorageLocation = String("some storage location") + removeAndAddOpts.AddStorageLocation = &ExternalVolumeStorageLocation{S3StorageLocationParams: s3StorageLocationParamsWithExternalId} - setAndAddOpts.Set = &AlterExternalVolumeSet{ AllowWrites: Bool(true), } - setAndAddOpts.AddStorageLocation = &ExternalVolumeStorageLocation{ S3StorageLocationParams: s3StorageLocationParamsWithExternalId, } + setAndAddOpts.Set = &AlterExternalVolumeSet{AllowWrites: Bool(true)} + setAndAddOpts.AddStorageLocation = &ExternalVolumeStorageLocation{S3StorageLocationParams: s3StorageLocationParamsWithExternalId} assertOptsInvalidJoinedErrors(t, removeAndSetOpts, errExactlyOneOf("AlterExternalVolumeOptions", "RemoveStorageLocation", "Set", "AddStorageLocation")) assertOptsInvalidJoinedErrors(t, removeAndAddOpts, errExactlyOneOf("AlterExternalVolumeOptions", "RemoveStorageLocation", "Set", "AddStorageLocation")) @@ -219,21 +216,21 @@ func TestExternalVolumes_Alter(t *testing.T) { t.Run("validation: exactly one field from [opts.RemoveStorageLocation opts.Set opts.AddStorageLocation] should be present - three set", func(t *testing.T) { opts := defaultOpts() - opts.RemoveStorageLocation = String("some storage location") - opts.Set = &AlterExternalVolumeSet{ AllowWrites: Bool(true), } - opts.AddStorageLocation = &ExternalVolumeStorageLocation{ S3StorageLocationParams: s3StorageLocationParamsWithExternalId, } + opts.RemoveStorageLocation = String("some storage location") + opts.Set = &AlterExternalVolumeSet{AllowWrites: Bool(true)} + opts.AddStorageLocation = &ExternalVolumeStorageLocation{S3StorageLocationParams: s3StorageLocationParamsWithExternalId} assertOptsInvalidJoinedErrors(t, opts, errExactlyOneOf("AlterExternalVolumeOptions", "RemoveStorageLocation", "Set", "AddStorageLocation")) }) t.Run("validation: valid identifier for [opts.name]", func(t *testing.T) { opts := defaultOpts() - opts.name = emptyAccountObjectIdentifier + opts.name = emptyAccountObjectIdentifier assertOptsInvalidJoinedErrors(t, opts, ErrInvalidObjectIdentifier) }) t.Run("validation: exactly one field from [opts.AddStorageLocation.S3StorageLocationParams opts.AddStorageLocation.GCSStorageLocationParams opts.AddStorageLocation.AzureStorageLocationParams] should be present - none set", func(t *testing.T) { opts := defaultOpts() - opts.AddStorageLocation = &ExternalVolumeStorageLocation{} + opts.AddStorageLocation = &ExternalVolumeStorageLocation{} assertOptsInvalidJoinedErrors(t, opts, errExactlyOneOf("AlterExternalVolumeOptions.AddStorageLocation", "S3StorageLocationParams", "GCSStorageLocationParams", "AzureStorageLocationParams")) }) @@ -241,18 +238,18 @@ func TestExternalVolumes_Alter(t *testing.T) { s3AndGcsOpts := defaultOpts() s3AndAzureOpts := defaultOpts() gcsAndAzureOpts := defaultOpts() - s3AndGcsOpts.AddStorageLocation = &ExternalVolumeStorageLocation{ - S3StorageLocationParams: s3StorageLocationParams, - GCSStorageLocationParams: gcsStorageLocationParams, - } - s3AndAzureOpts.AddStorageLocation = &ExternalVolumeStorageLocation{ - S3StorageLocationParams: s3StorageLocationParams, - AzureStorageLocationParams: azureStorageLocationParams, - } - gcsAndAzureOpts.AddStorageLocation = &ExternalVolumeStorageLocation{ - GCSStorageLocationParams: gcsStorageLocationParams, - AzureStorageLocationParams: azureStorageLocationParams, - } + s3AndGcsOpts.AddStorageLocation = &ExternalVolumeStorageLocation{ + S3StorageLocationParams: s3StorageLocationParams, + GCSStorageLocationParams: gcsStorageLocationParams, + } + s3AndAzureOpts.AddStorageLocation = &ExternalVolumeStorageLocation{ + S3StorageLocationParams: s3StorageLocationParams, + AzureStorageLocationParams: azureStorageLocationParams, + } + gcsAndAzureOpts.AddStorageLocation = &ExternalVolumeStorageLocation{ + GCSStorageLocationParams: gcsStorageLocationParams, + AzureStorageLocationParams: azureStorageLocationParams, + } assertOptsInvalidJoinedErrors(t, s3AndGcsOpts, errExactlyOneOf("AlterExternalVolumeOptions.AddStorageLocation", "S3StorageLocationParams", "GCSStorageLocationParams", "AzureStorageLocationParams")) assertOptsInvalidJoinedErrors(t, s3AndAzureOpts, errExactlyOneOf("AlterExternalVolumeOptions.AddStorageLocation", "S3StorageLocationParams", "GCSStorageLocationParams", "AzureStorageLocationParams")) assertOptsInvalidJoinedErrors(t, gcsAndAzureOpts, errExactlyOneOf("AlterExternalVolumeOptions.AddStorageLocation", "S3StorageLocationParams", "GCSStorageLocationParams", "AzureStorageLocationParams")) @@ -260,53 +257,52 @@ func TestExternalVolumes_Alter(t *testing.T) { t.Run("validation: exactly one field from [opts.AddStorageLocation.S3StorageLocationParams opts.AddStorageLocation.GCSStorageLocationParams opts.AddStorageLocation.AzureStorageLocationParams] should be present - three set", func(t *testing.T) { opts := defaultOpts() - opts.AddStorageLocation = &ExternalVolumeStorageLocation{ - S3StorageLocationParams: s3StorageLocationParams, - GCSStorageLocationParams: gcsStorageLocationParams, - AzureStorageLocationParams: azureStorageLocationParams, - } + opts.AddStorageLocation = &ExternalVolumeStorageLocation{ + S3StorageLocationParams: s3StorageLocationParams, + GCSStorageLocationParams: gcsStorageLocationParams, + AzureStorageLocationParams: azureStorageLocationParams, + } assertOptsInvalidJoinedErrors(t, opts, errExactlyOneOf("AlterExternalVolumeOptions.AddStorageLocation", "S3StorageLocationParams", "GCSStorageLocationParams", "AzureStorageLocationParams")) }) t.Run("remove storage location", func(t *testing.T) { opts := defaultOpts() - opts.RemoveStorageLocation = String("some storage location") + opts.RemoveStorageLocation = String("some storage location") assertOptsValidAndSQLEquals(t, opts, `ALTER EXTERNAL VOLUME %s REMOVE STORAGE_LOCATION 'some storage location'`, id.FullyQualifiedName()) }) t.Run("set - allow writes", func(t *testing.T) { opts := defaultOpts() - opts.Set = &AlterExternalVolumeSet{ AllowWrites: Bool(true), } + opts.Set = &AlterExternalVolumeSet{AllowWrites: Bool(true)} assertOptsValidAndSQLEquals(t, opts, `ALTER EXTERNAL VOLUME %s SET ALLOW_WRITES = true`, id.FullyQualifiedName()) }) t.Run("set - comment", func(t *testing.T) { opts := defaultOpts() - opts.Set = &AlterExternalVolumeSet{ Comment: String("some comment"), } + opts.Set = &AlterExternalVolumeSet{Comment: String("some comment")} assertOptsValidAndSQLEquals(t, opts, `ALTER EXTERNAL VOLUME %s SET COMMENT = 'some comment'`, id.FullyQualifiedName()) }) t.Run("add storage location - s3", func(t *testing.T) { opts := defaultOpts() - opts.AddStorageLocation = &ExternalVolumeStorageLocation{ S3StorageLocationParams: s3StorageLocationParamsWithExternalId, } + opts.AddStorageLocation = &ExternalVolumeStorageLocation{S3StorageLocationParams: s3StorageLocationParamsWithExternalId} assertOptsValidAndSQLEquals(t, opts, `ALTER EXTERNAL VOLUME %s ADD STORAGE_LOCATION = (NAME = 'some s3 name' STORAGE_PROVIDER = 'S3' STORAGE_AWS_ROLE_ARN = 'some s3 role arn' STORAGE_BASE_URL = 'some s3 base url' STORAGE_AWS_EXTERNAL_ID = 'some s3 external id' ENCRYPTION = (TYPE = 'AWS_SSE_S3' KMS_KEY_ID = 'some s3 kms key id'))`, id.FullyQualifiedName()) }) t.Run("add storage location - gcs", func(t *testing.T) { opts := defaultOpts() - opts.AddStorageLocation = &ExternalVolumeStorageLocation{ GCSStorageLocationParams: gcsStorageLocationParams, } + opts.AddStorageLocation = &ExternalVolumeStorageLocation{GCSStorageLocationParams: gcsStorageLocationParams} assertOptsValidAndSQLEquals(t, opts, `ALTER EXTERNAL VOLUME %s ADD STORAGE_LOCATION = (NAME = 'some gcs name' STORAGE_PROVIDER = 'GCS' STORAGE_BASE_URL = 'some gcs base url' ENCRYPTION = (TYPE = 'GCS_SSE_KMS' KMS_KEY_ID = 'some gcs kms key id'))`, id.FullyQualifiedName()) }) t.Run("add storage location - azure", func(t *testing.T) { opts := defaultOpts() - opts.AddStorageLocation = &ExternalVolumeStorageLocation{ AzureStorageLocationParams: azureStorageLocationParams, } + opts.AddStorageLocation = &ExternalVolumeStorageLocation{AzureStorageLocationParams: azureStorageLocationParams} assertOptsValidAndSQLEquals(t, opts, `ALTER EXTERNAL VOLUME %s ADD STORAGE_LOCATION = (NAME = 'some azure name' STORAGE_PROVIDER = 'AZURE' AZURE_TENANT_ID = 'some azure tenant id' STORAGE_BASE_URL = 'some azure base url')`, id.FullyQualifiedName()) }) } func TestExternalVolumes_Drop(t *testing.T) { - id := randomAccountObjectIdentifier() // Minimal valid DropExternalVolumeOptions defaultOpts := func() *DropExternalVolumeOptions { @@ -322,7 +318,7 @@ func TestExternalVolumes_Drop(t *testing.T) { t.Run("validation: valid identifier for [opts.name]", func(t *testing.T) { opts := defaultOpts() - opts.name = emptyAccountObjectIdentifier + opts.name = emptyAccountObjectIdentifier assertOptsInvalidJoinedErrors(t, opts, ErrInvalidObjectIdentifier) }) @@ -333,13 +329,12 @@ func TestExternalVolumes_Drop(t *testing.T) { t.Run("all options", func(t *testing.T) { opts := defaultOpts() - opts.IfExists = Bool(true) + opts.IfExists = Bool(true) assertOptsValidAndSQLEquals(t, opts, `DROP EXTERNAL VOLUME IF EXISTS %s`, id.FullyQualifiedName()) }) } func TestExternalVolumes_Describe(t *testing.T) { - id := randomAccountObjectIdentifier() // Minimal valid DescribeExternalVolumeOptions defaultOpts := func() *DescribeExternalVolumeOptions { @@ -354,7 +349,7 @@ func TestExternalVolumes_Describe(t *testing.T) { }) t.Run("validation: valid identifier for [opts.name]", func(t *testing.T) { opts := defaultOpts() - opts.name = emptyAccountObjectIdentifier + opts.name = emptyAccountObjectIdentifier assertOptsInvalidJoinedErrors(t, opts, ErrInvalidObjectIdentifier) }) @@ -365,7 +360,7 @@ func TestExternalVolumes_Describe(t *testing.T) { } func TestExternalVolumes_Show(t *testing.T) { - id := randomAccountObjectIdentifier() + id := randomAccountObjectIdentifier() // Minimal valid ShowExternalVolumeOptions defaultOpts := func() *ShowExternalVolumeOptions { return &ShowExternalVolumeOptions{} @@ -383,7 +378,7 @@ func TestExternalVolumes_Show(t *testing.T) { t.Run("all options", func(t *testing.T) { opts := defaultOpts() - opts.Like = &Like{ + opts.Like = &Like{ Pattern: String(id.Name()), } assertOptsValidAndSQLEquals(t, opts, "SHOW EXTERNAL VOLUMES LIKE '%s'", id.Name()) diff --git a/pkg/sdk/external_volumes_impl_gen.go b/pkg/sdk/external_volumes_impl_gen.go index 459fced135..004f53e10c 100644 --- a/pkg/sdk/external_volumes_impl_gen.go +++ b/pkg/sdk/external_volumes_impl_gen.go @@ -2,6 +2,7 @@ package sdk import ( "context" + "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/internal/collections" ) @@ -75,12 +76,10 @@ func (r *AlterExternalVolumeRequest) toOpts() *AlterExternalVolumeOptions { } if r.Set != nil { - opts.Set = &AlterExternalVolumeSet{ AllowWrites: r.Set.AllowWrites, Comment: r.Set.Comment, } - } if r.AddStorageLocation != nil { @@ -119,13 +118,11 @@ func (r *AlterExternalVolumeRequest) toOpts() *AlterExternalVolumeOptions { } if r.AddStorageLocation.AzureStorageLocationParams != nil { - opts.AddStorageLocation.AzureStorageLocationParams = &AzureStorageLocationParams{ Name: r.AddStorageLocation.AzureStorageLocationParams.Name, AzureTenantId: r.AddStorageLocation.AzureStorageLocationParams.AzureTenantId, StorageBaseUrl: r.AddStorageLocation.AzureStorageLocationParams.StorageBaseUrl, } - } } @@ -150,7 +147,7 @@ func (r *DescribeExternalVolumeRequest) toOpts() *DescribeExternalVolumeOptions func (r externalVolumeDescRow) convert() *ExternalVolumeProperty { return &ExternalVolumeProperty{ - Parent: r.ParentProperty, + Parent: r.ParentProperty, Name: r.Property, Type: r.PropertyType, Value: r.PropertyValue, @@ -167,8 +164,8 @@ func (r *ShowExternalVolumeRequest) toOpts() *ShowExternalVolumeOptions { func (r externalVolumeShowRow) convert() *ExternalVolume { return &ExternalVolume{ - Name: r.Name, - AllowWrites: r.AllowWrites, - Comment: r.Comment, - } + Name: r.Name, + AllowWrites: r.AllowWrites, + Comment: r.Comment, + } } diff --git a/pkg/sdk/external_volumes_validations_gen.go b/pkg/sdk/external_volumes_validations_gen.go index 248204d0bf..3d4d5eb03d 100644 --- a/pkg/sdk/external_volumes_validations_gen.go +++ b/pkg/sdk/external_volumes_validations_gen.go @@ -22,19 +22,19 @@ func (opts *CreateExternalVolumeOptions) validate() error { errs = append(errs, ErrInvalidObjectIdentifier) } - // Custom (not code generated) validations + // Custom (not code generated) validations - // Apply errExactlyOneOf to each element in storage locations list - for i,storageLocation := range opts.StorageLocations { - if !exactlyOneValueSet(storageLocation.S3StorageLocationParams, storageLocation.GCSStorageLocationParams, storageLocation.AzureStorageLocationParams) { - errs = append(errs, errExactlyOneOf(fmt.Sprintf("CreateExternalVolumeOptions.StorageLocation[%d]", i), "S3StorageLocationParams", "GCSStorageLocationParams", "AzureStorageLocationParams")) - } - } + // Apply errExactlyOneOf to each element in storage locations list + for i, storageLocation := range opts.StorageLocations { + if !exactlyOneValueSet(storageLocation.S3StorageLocationParams, storageLocation.GCSStorageLocationParams, storageLocation.AzureStorageLocationParams) { + errs = append(errs, errExactlyOneOf(fmt.Sprintf("CreateExternalVolumeOptions.StorageLocation[%d]", i), "S3StorageLocationParams", "GCSStorageLocationParams", "AzureStorageLocationParams")) + } + } - // Check the storage location list is not empty, as at least 1 storage location is required for an external volume - if len(opts.StorageLocations) == 0 { - errs = append(errs, errNotSet("CreateExternalVolumeOptions", "StorageLocations")) - } + // Check the storage location list is not empty, as at least 1 storage location is required for an external volume + if len(opts.StorageLocations) == 0 { + errs = append(errs, errNotSet("CreateExternalVolumeOptions", "StorageLocations")) + } return JoinErrors(errs...) } diff --git a/pkg/sdk/testint/external_volumes_gen_integration_test.go b/pkg/sdk/testint/external_volumes_gen_integration_test.go index 7a16924088..85ec0edc97 100644 --- a/pkg/sdk/testint/external_volumes_gen_integration_test.go +++ b/pkg/sdk/testint/external_volumes_gen_integration_test.go @@ -1,11 +1,12 @@ package testint import ( - "testing" + "encoding/json" + "fmt" "strconv" - "strings" - "fmt" - "encoding/json" + "strings" + "testing" + "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/acceptance/testenvs" "github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/sdk" "github.com/stretchr/testify/assert" @@ -18,214 +19,214 @@ func TestInt_ExternalVolumes(t *testing.T) { client := testClient(t) ctx := testContext(t) - awsBaseUrl := testenvs.GetOrSkipTest(t, testenvs.AwsExternalBucketUrl) - awsRoleARN := testenvs.GetOrSkipTest(t, testenvs.AwsExternalRoleArn) - awsKmsKeyId := testenvs.GetOrSkipTest(t, testenvs.AwsExternalKeyId) - awsExternalId := "123456789" + awsBaseUrl := testenvs.GetOrSkipTest(t, testenvs.AwsExternalBucketUrl) + awsRoleARN := testenvs.GetOrSkipTest(t, testenvs.AwsExternalRoleArn) + awsKmsKeyId := testenvs.GetOrSkipTest(t, testenvs.AwsExternalKeyId) + awsExternalId := "123456789" gcsBaseUrl := testenvs.GetOrSkipTest(t, testenvs.GcsExternalBuckerUrl) - // //TODO - To test external volumes wth gcs storage locations that have encryption TYPE=GCS_SSE_KMS a gcs KMS_KEY_ID is required, which is not currently available in testenvs + // //TODO - To test external volumes wth gcs storage locations that have encryption TYPE=GCS_SSE_KMS a gcs KMS_KEY_ID is required, which is not currently available in testenvs - azureBaseUrl := testenvs.GetOrSkipTest(t, testenvs.AzureExternalBucketUrl) // TODO verify assumption on form + azureBaseUrl := testenvs.GetOrSkipTest(t, testenvs.AzureExternalBucketUrl) // TODO verify assumption on form azureTenantId := testenvs.GetOrSkipTest(t, testenvs.AzureExternalTenantId) - assertExternalVolumeShowResult := func(t *testing.T, s *sdk.ExternalVolume, name sdk.AccountObjectIdentifier, allowWrites bool, comment string) { + assertExternalVolumeShowResult := func(t *testing.T, s *sdk.ExternalVolume, name sdk.AccountObjectIdentifier, allowWrites bool, comment string) { t.Helper() assert.Equal(t, name.Name(), s.Name) assert.Equal(t, strconv.FormatBool(allowWrites), s.AllowWrites) assert.Equal(t, comment, s.Comment) } - type ExternalVolumePropNameValue struct { - Name string - Value string - } - - type S3StorageLocation struct { - Name string `json:"NAME"` - StorageProvider string `json:"STORAGE_PROVIDER"` - StorageBaseUrl string `json:"STORAGE_BASE_URL"` - StorageAllowedLocations []string `json:"STORAGE_ALLOWED_LOCATIONS"` - StorageAwsRoleArn string `json:"STORAGE_AWS_ROLE_ARN"` - StroageAwsIamUserArn string `json:"STORAGE_AWS_IAM_USER_ARN"` - StorageAwsExternalId string `json:"STORAGE_AWS_EXTERNAL_ID"` - EncryptionType string `json:"ENCRYPTION_TYPE"` - EncryptionKmsId string `json:"ENCRYPTION_KMS_KEY_ID"` - } - - type S3StorageLocationTrimmed struct { - Name string `json:"NAME"` - StorageProvider string `json:"STORAGE_PROVIDER"` - StorageBaseUrl string `json:"STORAGE_BASE_URL"` - StorageAwsRoleArn string `json:"STORAGE_AWS_ROLE_ARN"` - StorageAwsExternalId string `json:"STORAGE_AWS_EXTERNAL_ID"` - EncryptionType string `json:"ENCRYPTION_TYPE"` - EncryptionKmsId string `json:"ENCRYPTION_KMS_KEY_ID,omitempty"` - } - - type GCSStorageLocation struct { - Name string `json:"NAME"` - StorageProvider string `json:"STORAGE_PROVIDER"` - StorageBaseUrl string `json:"STORAGE_BASE_URL"` - StorageAllowedLocations []string `json:"STORAGE_ALLOWED_LOCATIONS"` - StorageGcpServiceAccount string `json:"STORAGE_GCP_SERVICE_ACCOUNT"` - EncryptionType string `json:"ENCRYPTION_TYPE"` - EncryptionKmsId string `json:"ENCRYPTION_KMS_KEY_ID"` - } - - type GCSStorageLocationTrimmed struct { - Name string `json:"NAME"` - StorageProvider string `json:"STORAGE_PROVIDER"` - StorageBaseUrl string `json:"STORAGE_BASE_URL"` - EncryptionType string `json:"ENCRYPTION_TYPE"` - EncryptionKmsId string `json:"ENCRYPTION_KMS_KEY_ID,omitempty"` - } - - type AzureStorageLocation struct { - Name string `json:"NAME"` - StorageProvider string `json:"STORAGE_PROVIDER"` - StorageBaseUrl string `json:"STORAGE_BASE_URL"` - StorageAllowedLocations []string `json:"STORAGE_ALLOWED_LOCATIONS"` - AzureTenantId string `json:"AZURE_TENANT_ID"` - AzureMultiTenantAppName string `json:"AZURE_MULTI_TENANT_APP_NAME"` - AzureConsentUrl string `json:"AZURE_CONSENT_URL"` - EncryptionType string `json:"ENCRYPTION_TYPE"` - EncryptionKmsId string `json:"ENCRYPTION_KMS_KEY_ID"` - } - - type AzureStorageLocationTrimmed struct { - Name string `json:"NAME"` - StorageProvider string `json:"STORAGE_PROVIDER"` - StorageBaseUrl string `json:"STORAGE_BASE_URL"` - AzureTenantId string `json:"AZURE_TENANT_ID"` - } - - // Enforce only property names and values in tests, not parent_property, type and property_default - // In addition the storage location properties are trimmed to only contain values that we set - trimProperties := func(t *testing.T, props []sdk.ExternalVolumeProperty) []ExternalVolumePropNameValue { + type ExternalVolumePropNameValue struct { + Name string + Value string + } + + type S3StorageLocation struct { + Name string `json:"NAME"` + StorageProvider string `json:"STORAGE_PROVIDER"` + StorageBaseUrl string `json:"STORAGE_BASE_URL"` + StorageAllowedLocations []string `json:"STORAGE_ALLOWED_LOCATIONS"` + StorageAwsRoleArn string `json:"STORAGE_AWS_ROLE_ARN"` + StroageAwsIamUserArn string `json:"STORAGE_AWS_IAM_USER_ARN"` + StorageAwsExternalId string `json:"STORAGE_AWS_EXTERNAL_ID"` + EncryptionType string `json:"ENCRYPTION_TYPE"` + EncryptionKmsId string `json:"ENCRYPTION_KMS_KEY_ID"` + } + + type S3StorageLocationTrimmed struct { + Name string `json:"NAME"` + StorageProvider string `json:"STORAGE_PROVIDER"` + StorageBaseUrl string `json:"STORAGE_BASE_URL"` + StorageAwsRoleArn string `json:"STORAGE_AWS_ROLE_ARN"` + StorageAwsExternalId string `json:"STORAGE_AWS_EXTERNAL_ID"` + EncryptionType string `json:"ENCRYPTION_TYPE"` + EncryptionKmsId string `json:"ENCRYPTION_KMS_KEY_ID,omitempty"` + } + + type GCSStorageLocation struct { + Name string `json:"NAME"` + StorageProvider string `json:"STORAGE_PROVIDER"` + StorageBaseUrl string `json:"STORAGE_BASE_URL"` + StorageAllowedLocations []string `json:"STORAGE_ALLOWED_LOCATIONS"` + StorageGcpServiceAccount string `json:"STORAGE_GCP_SERVICE_ACCOUNT"` + EncryptionType string `json:"ENCRYPTION_TYPE"` + EncryptionKmsId string `json:"ENCRYPTION_KMS_KEY_ID"` + } + + type GCSStorageLocationTrimmed struct { + Name string `json:"NAME"` + StorageProvider string `json:"STORAGE_PROVIDER"` + StorageBaseUrl string `json:"STORAGE_BASE_URL"` + EncryptionType string `json:"ENCRYPTION_TYPE"` + EncryptionKmsId string `json:"ENCRYPTION_KMS_KEY_ID,omitempty"` + } + + type AzureStorageLocation struct { + Name string `json:"NAME"` + StorageProvider string `json:"STORAGE_PROVIDER"` + StorageBaseUrl string `json:"STORAGE_BASE_URL"` + StorageAllowedLocations []string `json:"STORAGE_ALLOWED_LOCATIONS"` + AzureTenantId string `json:"AZURE_TENANT_ID"` + AzureMultiTenantAppName string `json:"AZURE_MULTI_TENANT_APP_NAME"` + AzureConsentUrl string `json:"AZURE_CONSENT_URL"` + EncryptionType string `json:"ENCRYPTION_TYPE"` + EncryptionKmsId string `json:"ENCRYPTION_KMS_KEY_ID"` + } + + type AzureStorageLocationTrimmed struct { + Name string `json:"NAME"` + StorageProvider string `json:"STORAGE_PROVIDER"` + StorageBaseUrl string `json:"STORAGE_BASE_URL"` + AzureTenantId string `json:"AZURE_TENANT_ID"` + } + + // Enforce only property names and values in tests, not parent_property, type and property_default + // In addition the storage location properties are trimmed to only contain values that we set + trimProperties := func(t *testing.T, props []sdk.ExternalVolumeProperty) []ExternalVolumePropNameValue { t.Helper() - var externalVolumePropNameValue []ExternalVolumePropNameValue - for _,p := range props { - if strings.Contains(p.Name, "STORAGE_LOCATION_") { - if strings.Contains(p.Value, `"STORAGE_PROVIDER":"S3"`) { - s3StorageLocation := S3StorageLocation{} - json.Unmarshal([]byte(p.Value), &s3StorageLocation) - s3StorageLocationTrimmed := S3StorageLocationTrimmed{ - Name: s3StorageLocation.Name, - StorageProvider: s3StorageLocation.StorageProvider, - StorageBaseUrl: s3StorageLocation.StorageBaseUrl, - StorageAwsRoleArn: s3StorageLocation.StorageAwsRoleArn, - StorageAwsExternalId: s3StorageLocation.StorageAwsExternalId, - EncryptionType: s3StorageLocation.EncryptionType, - EncryptionKmsId: s3StorageLocation.EncryptionKmsId, - } - s3StorageLocationTrimmedMarshaled, err := json.Marshal(s3StorageLocationTrimmed) - require.NoError(t, err) - externalVolumePropNameValue = append( - externalVolumePropNameValue, - ExternalVolumePropNameValue{Name: p.Name, Value: string(s3StorageLocationTrimmedMarshaled)}, - ) - } else if strings.Contains(p.Value, `"STORAGE_PROVIDER":"GCS"`) { - gcsStorageLocation := GCSStorageLocation{} - json.Unmarshal([]byte(p.Value), &gcsStorageLocation) - gcsStorageLocationTrimmed := GCSStorageLocationTrimmed{ - Name: gcsStorageLocation.Name, - StorageProvider: gcsStorageLocation.StorageProvider, - StorageBaseUrl: gcsStorageLocation.StorageBaseUrl, - EncryptionType: gcsStorageLocation.EncryptionType, - EncryptionKmsId: gcsStorageLocation.EncryptionKmsId, - } - gcsStorageLocationTrimmedMarshaled, err := json.Marshal(gcsStorageLocationTrimmed) - require.NoError(t, err) - externalVolumePropNameValue = append( - externalVolumePropNameValue, - ExternalVolumePropNameValue{Name: p.Name, Value: string(gcsStorageLocationTrimmedMarshaled)}, - ) - } else if strings.Contains(p.Value, `"STORAGE_PROVIDER":"AZURE"`) { - azureStorageLocation := AzureStorageLocation{} - json.Unmarshal([]byte(p.Value), &azureStorageLocation) - azureStorageLocationTrimmed := AzureStorageLocationTrimmed{ - Name: azureStorageLocation.Name, - StorageProvider: azureStorageLocation.StorageProvider, - StorageBaseUrl: azureStorageLocation.StorageBaseUrl, - AzureTenantId: azureStorageLocation.AzureTenantId, - } - azureStorageLocationTrimmedMarshaled, err := json.Marshal(azureStorageLocationTrimmed) - require.NoError(t, err) - externalVolumePropNameValue = append( - externalVolumePropNameValue, - ExternalVolumePropNameValue{Name: p.Name, Value: string(azureStorageLocationTrimmedMarshaled)}, - ) - } else { - panic("Unrecognised storage provider in storage location property") - } - } else { - externalVolumePropNameValue = append(externalVolumePropNameValue, ExternalVolumePropNameValue{Name: p.Name, Value: p.Value}) - } - } - - return externalVolumePropNameValue + var externalVolumePropNameValue []ExternalVolumePropNameValue + for _, p := range props { + if strings.Contains(p.Name, "STORAGE_LOCATION_") { + if strings.Contains(p.Value, `"STORAGE_PROVIDER":"S3"`) { + s3StorageLocation := S3StorageLocation{} + json.Unmarshal([]byte(p.Value), &s3StorageLocation) + s3StorageLocationTrimmed := S3StorageLocationTrimmed{ + Name: s3StorageLocation.Name, + StorageProvider: s3StorageLocation.StorageProvider, + StorageBaseUrl: s3StorageLocation.StorageBaseUrl, + StorageAwsRoleArn: s3StorageLocation.StorageAwsRoleArn, + StorageAwsExternalId: s3StorageLocation.StorageAwsExternalId, + EncryptionType: s3StorageLocation.EncryptionType, + EncryptionKmsId: s3StorageLocation.EncryptionKmsId, + } + s3StorageLocationTrimmedMarshaled, err := json.Marshal(s3StorageLocationTrimmed) + require.NoError(t, err) + externalVolumePropNameValue = append( + externalVolumePropNameValue, + ExternalVolumePropNameValue{Name: p.Name, Value: string(s3StorageLocationTrimmedMarshaled)}, + ) + } else if strings.Contains(p.Value, `"STORAGE_PROVIDER":"GCS"`) { + gcsStorageLocation := GCSStorageLocation{} + json.Unmarshal([]byte(p.Value), &gcsStorageLocation) + gcsStorageLocationTrimmed := GCSStorageLocationTrimmed{ + Name: gcsStorageLocation.Name, + StorageProvider: gcsStorageLocation.StorageProvider, + StorageBaseUrl: gcsStorageLocation.StorageBaseUrl, + EncryptionType: gcsStorageLocation.EncryptionType, + EncryptionKmsId: gcsStorageLocation.EncryptionKmsId, + } + gcsStorageLocationTrimmedMarshaled, err := json.Marshal(gcsStorageLocationTrimmed) + require.NoError(t, err) + externalVolumePropNameValue = append( + externalVolumePropNameValue, + ExternalVolumePropNameValue{Name: p.Name, Value: string(gcsStorageLocationTrimmedMarshaled)}, + ) + } else if strings.Contains(p.Value, `"STORAGE_PROVIDER":"AZURE"`) { + azureStorageLocation := AzureStorageLocation{} + json.Unmarshal([]byte(p.Value), &azureStorageLocation) + azureStorageLocationTrimmed := AzureStorageLocationTrimmed{ + Name: azureStorageLocation.Name, + StorageProvider: azureStorageLocation.StorageProvider, + StorageBaseUrl: azureStorageLocation.StorageBaseUrl, + AzureTenantId: azureStorageLocation.AzureTenantId, + } + azureStorageLocationTrimmedMarshaled, err := json.Marshal(azureStorageLocationTrimmed) + require.NoError(t, err) + externalVolumePropNameValue = append( + externalVolumePropNameValue, + ExternalVolumePropNameValue{Name: p.Name, Value: string(azureStorageLocationTrimmedMarshaled)}, + ) + } else { + panic("Unrecognised storage provider in storage location property") + } + } else { + externalVolumePropNameValue = append(externalVolumePropNameValue, ExternalVolumePropNameValue{Name: p.Name, Value: p.Value}) + } + } + + return externalVolumePropNameValue + } + + s3StorageLocations := []sdk.ExternalVolumeStorageLocation{ + { + S3StorageLocationParams: &sdk.S3StorageLocationParams{ + Name: "s3_testing_storage_location", + StorageProvider: sdk.S3StorageProviderS3, + StorageAwsRoleArn: awsRoleARN, + StorageBaseUrl: awsBaseUrl, + StorageAwsExternalId: sdk.String(awsExternalId), + Encryption: sdk.ExternalVolumeS3Encryption{ + Type: sdk.S3EncryptionTypeSseKms, + KmsKeyId: &awsKmsKeyId, + }, + }, + }, + } + + s3StorageLocationsNoneEncryption := []sdk.ExternalVolumeStorageLocation{ + { + S3StorageLocationParams: &sdk.S3StorageLocationParams{ + Name: "s3_testing_storage_location_no_encryption", + StorageProvider: sdk.S3StorageProviderS3, + StorageAwsRoleArn: awsRoleARN, + StorageBaseUrl: awsBaseUrl, + StorageAwsExternalId: sdk.String(awsExternalId), + Encryption: sdk.ExternalVolumeS3Encryption{ + Type: sdk.S3EncryptionNone, + }, + }, + }, + } + + gcsStorageLocations := []sdk.ExternalVolumeStorageLocation{ + { + GCSStorageLocationParams: &sdk.GCSStorageLocationParams{ + Name: "gcs_testing_storage_location", + StorageBaseUrl: gcsBaseUrl, + Encryption: sdk.ExternalVolumeGCSEncryption{ + Type: sdk.GCSEncryptionTypeNone, + }, + }, + }, + } + + azureStorageLocations := []sdk.ExternalVolumeStorageLocation{ + { + AzureStorageLocationParams: &sdk.AzureStorageLocationParams{ + Name: "azure_testing_storage_location", + AzureTenantId: azureTenantId, + StorageBaseUrl: azureBaseUrl, + }, + }, } - s3StorageLocations := []sdk.ExternalVolumeStorageLocation{ - { - S3StorageLocationParams: &sdk.S3StorageLocationParams{ - Name: "s3_testing_storage_location", - StorageProvider: sdk.S3StorageProviderS3, - StorageAwsRoleArn: awsRoleARN, - StorageBaseUrl: awsBaseUrl, - StorageAwsExternalId: sdk.String(awsExternalId), - Encryption: sdk.ExternalVolumeS3Encryption{ - Type: sdk.S3EncryptionTypeSseKms, - KmsKeyId: &awsKmsKeyId, - }, - }, - }, - } - - s3StorageLocationsNoneEncryption := []sdk.ExternalVolumeStorageLocation{ - { - S3StorageLocationParams: &sdk.S3StorageLocationParams{ - Name: "s3_testing_storage_location_no_encryption", - StorageProvider: sdk.S3StorageProviderS3, - StorageAwsRoleArn: awsRoleARN, - StorageBaseUrl: awsBaseUrl, - StorageAwsExternalId: sdk.String(awsExternalId), - Encryption: sdk.ExternalVolumeS3Encryption{ - Type: sdk.S3EncryptionNone, - }, - }, - }, - } - - gcsStorageLocations := []sdk.ExternalVolumeStorageLocation{ - { - GCSStorageLocationParams: &sdk.GCSStorageLocationParams{ - Name: "gcs_testing_storage_location", - StorageBaseUrl: gcsBaseUrl, - Encryption: sdk.ExternalVolumeGCSEncryption{ - Type: sdk.GCSEncryptionTypeNone, - }, - }, - }, - } - - azureStorageLocations := []sdk.ExternalVolumeStorageLocation{ - { - AzureStorageLocationParams: &sdk.AzureStorageLocationParams{ - Name: "azure_testing_storage_location", - AzureTenantId: azureTenantId, - StorageBaseUrl: azureBaseUrl, - }, - }, - } - - createExternalVolume:= func(t *testing.T, storageLocations []sdk.ExternalVolumeStorageLocation, allowWrites bool, comment string) sdk.AccountObjectIdentifier { + createExternalVolume := func(t *testing.T, storageLocations []sdk.ExternalVolumeStorageLocation, allowWrites bool, comment string) sdk.AccountObjectIdentifier { t.Helper() - //id := testClientHelper().Ids.RandomAccountObjectIdentifier() - //TODO remove this - id := sdk.NewAccountObjectIdentifier("test_external_volume") + // id := testClientHelper().Ids.RandomAccountObjectIdentifier() + // TODO remove this + id := sdk.NewAccountObjectIdentifier("test_external_volume") req := sdk.NewCreateExternalVolumeRequest(id, storageLocations). WithIfNotExists(true). WithAllowWrites(allowWrites). @@ -243,8 +244,8 @@ func TestInt_ExternalVolumes(t *testing.T) { } t.Run("Create - S3 Storage Location", func(t *testing.T) { - allowWrites := true - comment := "some comment" + allowWrites := true + comment := "some comment" id := createExternalVolume(t, s3StorageLocations, allowWrites, comment) externalVolume, err := client.ExternalVolumes.ShowByID(ctx, id) @@ -254,8 +255,8 @@ func TestInt_ExternalVolumes(t *testing.T) { }) t.Run("Create - S3 Storage Location No Encryption", func(t *testing.T) { - allowWrites := true - comment := "some comment" + allowWrites := true + comment := "some comment" id := createExternalVolume(t, s3StorageLocationsNoneEncryption, allowWrites, comment) externalVolume, err := client.ExternalVolumes.ShowByID(ctx, id) @@ -265,8 +266,8 @@ func TestInt_ExternalVolumes(t *testing.T) { }) t.Run("Create - GCS Storage Location", func(t *testing.T) { - allowWrites := true - comment := "some comment" + allowWrites := true + comment := "some comment" id := createExternalVolume(t, gcsStorageLocations, allowWrites, comment) externalVolume, err := client.ExternalVolumes.ShowByID(ctx, id) @@ -276,8 +277,8 @@ func TestInt_ExternalVolumes(t *testing.T) { }) t.Run("Create - Azure Storage Location", func(t *testing.T) { - allowWrites := true - comment := "some comment" + allowWrites := true + comment := "some comment" id := createExternalVolume(t, azureStorageLocations, allowWrites, comment) externalVolume, err := client.ExternalVolumes.ShowByID(ctx, id) @@ -287,8 +288,8 @@ func TestInt_ExternalVolumes(t *testing.T) { }) t.Run("Create - Multiple Storage Locations", func(t *testing.T) { - allowWrites := true - comment := "some comment" + allowWrites := true + comment := "some comment" id := createExternalVolume(t, append(append(s3StorageLocations, gcsStorageLocations...), azureStorageLocations...), allowWrites, comment) externalVolume, err := client.ExternalVolumes.ShowByID(ctx, id) @@ -298,8 +299,8 @@ func TestInt_ExternalVolumes(t *testing.T) { }) t.Run("Alter - remove storage location", func(t *testing.T) { - allowWrites := true - comment := "some comment" + allowWrites := true + comment := "some comment" id := createExternalVolume(t, append(s3StorageLocationsNoneEncryption, gcsStorageLocations...), allowWrites, comment) req := sdk.NewAlterExternalVolumeRequest(id).WithRemoveStorageLocation(gcsStorageLocations[0].GCSStorageLocationParams.Name) @@ -310,37 +311,37 @@ func TestInt_ExternalVolumes(t *testing.T) { props, err := client.ExternalVolumes.Describe(ctx, id) require.NoError(t, err) - trimmedProperties := trimProperties(t, props) + trimmedProperties := trimProperties(t, props) assert.Equal(t, 4, len(trimmedProperties)) - assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "COMMENT", Value: comment}) - assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "ALLOW_WRITES", Value: strconv.FormatBool(allowWrites)}) - assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "ACTIVE", Value: ""}) - assert.Contains( - t, - trimmedProperties, - ExternalVolumePropNameValue{ - Name: "STORAGE_LOCATION_1", - Value: fmt.Sprintf( - `{"NAME":"%s","STORAGE_PROVIDER":"%s","STORAGE_BASE_URL":"%s","STORAGE_AWS_ROLE_ARN":"%s","STORAGE_AWS_EXTERNAL_ID":"%s","ENCRYPTION_TYPE":"%s"}`, - s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.Name, - s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageProvider, - s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageBaseUrl, - s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageAwsRoleArn, - *s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageAwsExternalId, - s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.Encryption.Type, - ), - }, - ) + assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "COMMENT", Value: comment}) + assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "ALLOW_WRITES", Value: strconv.FormatBool(allowWrites)}) + assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "ACTIVE", Value: ""}) + assert.Contains( + t, + trimmedProperties, + ExternalVolumePropNameValue{ + Name: "STORAGE_LOCATION_1", + Value: fmt.Sprintf( + `{"NAME":"%s","STORAGE_PROVIDER":"%s","STORAGE_BASE_URL":"%s","STORAGE_AWS_ROLE_ARN":"%s","STORAGE_AWS_EXTERNAL_ID":"%s","ENCRYPTION_TYPE":"%s"}`, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.Name, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageProvider, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageBaseUrl, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageAwsRoleArn, + *s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageAwsExternalId, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.Encryption.Type, + ), + }, + ) }) t.Run("Alter - set comment", func(t *testing.T) { - allowWrites := true - comment := "" + allowWrites := true + comment := "" id := createExternalVolume(t, s3StorageLocationsNoneEncryption, allowWrites, "some comment") req := sdk.NewAlterExternalVolumeRequest(id).WithSet( - *sdk.NewAlterExternalVolumeSetRequest().WithComment(comment), - ) + *sdk.NewAlterExternalVolumeSetRequest().WithComment(comment), + ) err := client.ExternalVolumes.Alter(ctx, req) require.NoError(t, err) @@ -348,36 +349,36 @@ func TestInt_ExternalVolumes(t *testing.T) { props, err := client.ExternalVolumes.Describe(ctx, id) require.NoError(t, err) - trimmedProperties := trimProperties(t, props) + trimmedProperties := trimProperties(t, props) assert.Equal(t, 3, len(trimmedProperties)) - assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "ALLOW_WRITES", Value: strconv.FormatBool(allowWrites)}) - assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "ACTIVE", Value: ""}) - assert.Contains( - t, - trimmedProperties, - ExternalVolumePropNameValue{ - Name: "STORAGE_LOCATION_1", - Value: fmt.Sprintf( - `{"NAME":"%s","STORAGE_PROVIDER":"%s","STORAGE_BASE_URL":"%s","STORAGE_AWS_ROLE_ARN":"%s","STORAGE_AWS_EXTERNAL_ID":"%s","ENCRYPTION_TYPE":"%s"}`, - s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.Name, - s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageProvider, - s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageBaseUrl, - s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageAwsRoleArn, - *s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageAwsExternalId, - s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.Encryption.Type, - ), - }, - ) + assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "ALLOW_WRITES", Value: strconv.FormatBool(allowWrites)}) + assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "ACTIVE", Value: ""}) + assert.Contains( + t, + trimmedProperties, + ExternalVolumePropNameValue{ + Name: "STORAGE_LOCATION_1", + Value: fmt.Sprintf( + `{"NAME":"%s","STORAGE_PROVIDER":"%s","STORAGE_BASE_URL":"%s","STORAGE_AWS_ROLE_ARN":"%s","STORAGE_AWS_EXTERNAL_ID":"%s","ENCRYPTION_TYPE":"%s"}`, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.Name, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageProvider, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageBaseUrl, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageAwsRoleArn, + *s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageAwsExternalId, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.Encryption.Type, + ), + }, + ) }) t.Run("Alter - set allow writes", func(t *testing.T) { - allowWrites := false - comment := "some comment" + allowWrites := false + comment := "some comment" id := createExternalVolume(t, s3StorageLocations, true, comment) req := sdk.NewAlterExternalVolumeRequest(id).WithSet( - *sdk.NewAlterExternalVolumeSetRequest().WithAllowWrites(allowWrites), - ) + *sdk.NewAlterExternalVolumeSetRequest().WithAllowWrites(allowWrites), + ) err := client.ExternalVolumes.Alter(ctx, req) require.NoError(t, err) @@ -385,48 +386,48 @@ func TestInt_ExternalVolumes(t *testing.T) { props, err := client.ExternalVolumes.Describe(ctx, id) require.NoError(t, err) - trimmedProperties := trimProperties(t, props) + trimmedProperties := trimProperties(t, props) assert.Equal(t, 4, len(trimmedProperties)) - assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "COMMENT", Value: comment}) - assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "ALLOW_WRITES", Value: strconv.FormatBool(allowWrites)}) - assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "ACTIVE", Value: ""}) - assert.Contains( - t, - trimmedProperties, - ExternalVolumePropNameValue{ - Name: "STORAGE_LOCATION_1", - Value: fmt.Sprintf( - `{"NAME":"%s","STORAGE_PROVIDER":"%s","STORAGE_BASE_URL":"%s","STORAGE_AWS_ROLE_ARN":"%s","STORAGE_AWS_EXTERNAL_ID":"%s","ENCRYPTION_TYPE":"%s","ENCRYPTION_KMS_KEY_ID":"%s"}`, - s3StorageLocations[0].S3StorageLocationParams.Name, - s3StorageLocations[0].S3StorageLocationParams.StorageProvider, - s3StorageLocations[0].S3StorageLocationParams.StorageBaseUrl, - s3StorageLocations[0].S3StorageLocationParams.StorageAwsRoleArn, - *s3StorageLocations[0].S3StorageLocationParams.StorageAwsExternalId, - s3StorageLocations[0].S3StorageLocationParams.Encryption.Type, - *s3StorageLocations[0].S3StorageLocationParams.Encryption.KmsKeyId, - ), - }, - ) + assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "COMMENT", Value: comment}) + assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "ALLOW_WRITES", Value: strconv.FormatBool(allowWrites)}) + assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "ACTIVE", Value: ""}) + assert.Contains( + t, + trimmedProperties, + ExternalVolumePropNameValue{ + Name: "STORAGE_LOCATION_1", + Value: fmt.Sprintf( + `{"NAME":"%s","STORAGE_PROVIDER":"%s","STORAGE_BASE_URL":"%s","STORAGE_AWS_ROLE_ARN":"%s","STORAGE_AWS_EXTERNAL_ID":"%s","ENCRYPTION_TYPE":"%s","ENCRYPTION_KMS_KEY_ID":"%s"}`, + s3StorageLocations[0].S3StorageLocationParams.Name, + s3StorageLocations[0].S3StorageLocationParams.StorageProvider, + s3StorageLocations[0].S3StorageLocationParams.StorageBaseUrl, + s3StorageLocations[0].S3StorageLocationParams.StorageAwsRoleArn, + *s3StorageLocations[0].S3StorageLocationParams.StorageAwsExternalId, + s3StorageLocations[0].S3StorageLocationParams.Encryption.Type, + *s3StorageLocations[0].S3StorageLocationParams.Encryption.KmsKeyId, + ), + }, + ) }) t.Run("Alter - add s3 storage location to external volume", func(t *testing.T) { - allowWrites := true - comment := "some comment" + allowWrites := true + comment := "some comment" id := createExternalVolume(t, gcsStorageLocations, allowWrites, comment) req := sdk.NewAlterExternalVolumeRequest(id).WithAddStorageLocation( - *sdk.NewExternalVolumeStorageLocationRequest().WithS3StorageLocationParams( - *sdk.NewS3StorageLocationParamsRequest( - s3StorageLocations[0].S3StorageLocationParams.Name, - s3StorageLocations[0].S3StorageLocationParams.StorageProvider, - s3StorageLocations[0].S3StorageLocationParams.StorageAwsRoleArn, - s3StorageLocations[0].S3StorageLocationParams.StorageBaseUrl, - *sdk.NewExternalVolumeS3EncryptionRequest( - s3StorageLocations[0].S3StorageLocationParams.Encryption.Type, - ).WithKmsKeyId(*s3StorageLocations[0].S3StorageLocationParams.Encryption.KmsKeyId,), - ).WithStorageAwsExternalId(*s3StorageLocations[0].S3StorageLocationParams.StorageAwsExternalId,), - ), - ) + *sdk.NewExternalVolumeStorageLocationRequest().WithS3StorageLocationParams( + *sdk.NewS3StorageLocationParamsRequest( + s3StorageLocations[0].S3StorageLocationParams.Name, + s3StorageLocations[0].S3StorageLocationParams.StorageProvider, + s3StorageLocations[0].S3StorageLocationParams.StorageAwsRoleArn, + s3StorageLocations[0].S3StorageLocationParams.StorageBaseUrl, + *sdk.NewExternalVolumeS3EncryptionRequest( + s3StorageLocations[0].S3StorageLocationParams.Encryption.Type, + ).WithKmsKeyId(*s3StorageLocations[0].S3StorageLocationParams.Encryption.KmsKeyId), + ).WithStorageAwsExternalId(*s3StorageLocations[0].S3StorageLocationParams.StorageAwsExternalId), + ), + ) err := client.ExternalVolumes.Alter(ctx, req) require.NoError(t, err) @@ -434,128 +435,128 @@ func TestInt_ExternalVolumes(t *testing.T) { props, err := client.ExternalVolumes.Describe(ctx, id) require.NoError(t, err) - trimmedProperties := trimProperties(t, props) + trimmedProperties := trimProperties(t, props) assert.Equal(t, 5, len(trimmedProperties)) - assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "COMMENT", Value: comment}) - assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "ALLOW_WRITES", Value: strconv.FormatBool(allowWrites)}) - assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "ACTIVE", Value: ""}) - assert.Contains( - t, - trimmedProperties, - ExternalVolumePropNameValue{ - Name: "STORAGE_LOCATION_1", - Value: fmt.Sprintf( - `{"NAME":"%s","STORAGE_PROVIDER":"GCS","STORAGE_BASE_URL":"%s","ENCRYPTION_TYPE":"%s"}`, - gcsStorageLocations[0].GCSStorageLocationParams.Name, - gcsStorageLocations[0].GCSStorageLocationParams.StorageBaseUrl, - gcsStorageLocations[0].GCSStorageLocationParams.Encryption.Type, - ), - }, - ) - assert.Contains( - t, - trimmedProperties, - ExternalVolumePropNameValue{ - Name: "STORAGE_LOCATION_2", - Value: fmt.Sprintf( - `{"NAME":"%s","STORAGE_PROVIDER":"%s","STORAGE_BASE_URL":"%s","STORAGE_AWS_ROLE_ARN":"%s","STORAGE_AWS_EXTERNAL_ID":"%s","ENCRYPTION_TYPE":"%s","ENCRYPTION_KMS_KEY_ID":"%s"}`, - s3StorageLocations[0].S3StorageLocationParams.Name, - s3StorageLocations[0].S3StorageLocationParams.StorageProvider, - s3StorageLocations[0].S3StorageLocationParams.StorageBaseUrl, - s3StorageLocations[0].S3StorageLocationParams.StorageAwsRoleArn, - *s3StorageLocations[0].S3StorageLocationParams.StorageAwsExternalId, - s3StorageLocations[0].S3StorageLocationParams.Encryption.Type, - *s3StorageLocations[0].S3StorageLocationParams.Encryption.KmsKeyId, - ), - }, - ) + assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "COMMENT", Value: comment}) + assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "ALLOW_WRITES", Value: strconv.FormatBool(allowWrites)}) + assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "ACTIVE", Value: ""}) + assert.Contains( + t, + trimmedProperties, + ExternalVolumePropNameValue{ + Name: "STORAGE_LOCATION_1", + Value: fmt.Sprintf( + `{"NAME":"%s","STORAGE_PROVIDER":"GCS","STORAGE_BASE_URL":"%s","ENCRYPTION_TYPE":"%s"}`, + gcsStorageLocations[0].GCSStorageLocationParams.Name, + gcsStorageLocations[0].GCSStorageLocationParams.StorageBaseUrl, + gcsStorageLocations[0].GCSStorageLocationParams.Encryption.Type, + ), + }, + ) + assert.Contains( + t, + trimmedProperties, + ExternalVolumePropNameValue{ + Name: "STORAGE_LOCATION_2", + Value: fmt.Sprintf( + `{"NAME":"%s","STORAGE_PROVIDER":"%s","STORAGE_BASE_URL":"%s","STORAGE_AWS_ROLE_ARN":"%s","STORAGE_AWS_EXTERNAL_ID":"%s","ENCRYPTION_TYPE":"%s","ENCRYPTION_KMS_KEY_ID":"%s"}`, + s3StorageLocations[0].S3StorageLocationParams.Name, + s3StorageLocations[0].S3StorageLocationParams.StorageProvider, + s3StorageLocations[0].S3StorageLocationParams.StorageBaseUrl, + s3StorageLocations[0].S3StorageLocationParams.StorageAwsRoleArn, + *s3StorageLocations[0].S3StorageLocationParams.StorageAwsExternalId, + s3StorageLocations[0].S3StorageLocationParams.Encryption.Type, + *s3StorageLocations[0].S3StorageLocationParams.Encryption.KmsKeyId, + ), + }, + ) }) t.Run("Describe", func(t *testing.T) { - allowWrites := true - comment := "some comment" + allowWrites := true + comment := "some comment" id := createExternalVolume( - t, - append(append(append(s3StorageLocations, gcsStorageLocations...), azureStorageLocations...), s3StorageLocationsNoneEncryption...), - allowWrites, - comment, - ) + t, + append(append(append(s3StorageLocations, gcsStorageLocations...), azureStorageLocations...), s3StorageLocationsNoneEncryption...), + allowWrites, + comment, + ) props, err := client.ExternalVolumes.Describe(ctx, id) require.NoError(t, err) - trimmedProperties := trimProperties(t, props) + trimmedProperties := trimProperties(t, props) assert.Equal(t, 7, len(trimmedProperties)) - assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "COMMENT", Value: comment}) - assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "ALLOW_WRITES", Value: strconv.FormatBool(allowWrites)}) - assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "ACTIVE", Value: ""}) - assert.Contains( - t, - trimmedProperties, - ExternalVolumePropNameValue{ - Name: "STORAGE_LOCATION_1", - Value: fmt.Sprintf( - `{"NAME":"%s","STORAGE_PROVIDER":"%s","STORAGE_BASE_URL":"%s","STORAGE_AWS_ROLE_ARN":"%s","STORAGE_AWS_EXTERNAL_ID":"%s","ENCRYPTION_TYPE":"%s","ENCRYPTION_KMS_KEY_ID":"%s"}`, - s3StorageLocations[0].S3StorageLocationParams.Name, - s3StorageLocations[0].S3StorageLocationParams.StorageProvider, - s3StorageLocations[0].S3StorageLocationParams.StorageBaseUrl, - s3StorageLocations[0].S3StorageLocationParams.StorageAwsRoleArn, - *s3StorageLocations[0].S3StorageLocationParams.StorageAwsExternalId, - s3StorageLocations[0].S3StorageLocationParams.Encryption.Type, - *s3StorageLocations[0].S3StorageLocationParams.Encryption.KmsKeyId, - ), - }, - ) - assert.Contains( - t, - trimmedProperties, - ExternalVolumePropNameValue{ - Name: "STORAGE_LOCATION_2", - Value: fmt.Sprintf( - `{"NAME":"%s","STORAGE_PROVIDER":"GCS","STORAGE_BASE_URL":"%s","ENCRYPTION_TYPE":"%s"}`, - gcsStorageLocations[0].GCSStorageLocationParams.Name, - gcsStorageLocations[0].GCSStorageLocationParams.StorageBaseUrl, - gcsStorageLocations[0].GCSStorageLocationParams.Encryption.Type, - ), - }, - ) - assert.Contains( - t, - trimmedProperties, - ExternalVolumePropNameValue{ - Name: "STORAGE_LOCATION_3", - Value: fmt.Sprintf( - `{"NAME":"%s","STORAGE_PROVIDER":"AZURE","STORAGE_BASE_URL":"%s","AZURE_TENANT_ID":"%s"}`, - azureStorageLocations[0].AzureStorageLocationParams.Name, - azureStorageLocations[0].AzureStorageLocationParams.StorageBaseUrl, - azureStorageLocations[0].AzureStorageLocationParams.AzureTenantId, - ), - }, - ) - assert.Contains( - t, - trimmedProperties, - ExternalVolumePropNameValue{ - Name: "STORAGE_LOCATION_4", - Value: fmt.Sprintf( - `{"NAME":"%s","STORAGE_PROVIDER":"%s","STORAGE_BASE_URL":"%s","STORAGE_AWS_ROLE_ARN":"%s","STORAGE_AWS_EXTERNAL_ID":"%s","ENCRYPTION_TYPE":"%s"}`, - s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.Name, - s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageProvider, - s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageBaseUrl, - s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageAwsRoleArn, - *s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageAwsExternalId, - s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.Encryption.Type, - ), - }, - ) + assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "COMMENT", Value: comment}) + assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "ALLOW_WRITES", Value: strconv.FormatBool(allowWrites)}) + assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "ACTIVE", Value: ""}) + assert.Contains( + t, + trimmedProperties, + ExternalVolumePropNameValue{ + Name: "STORAGE_LOCATION_1", + Value: fmt.Sprintf( + `{"NAME":"%s","STORAGE_PROVIDER":"%s","STORAGE_BASE_URL":"%s","STORAGE_AWS_ROLE_ARN":"%s","STORAGE_AWS_EXTERNAL_ID":"%s","ENCRYPTION_TYPE":"%s","ENCRYPTION_KMS_KEY_ID":"%s"}`, + s3StorageLocations[0].S3StorageLocationParams.Name, + s3StorageLocations[0].S3StorageLocationParams.StorageProvider, + s3StorageLocations[0].S3StorageLocationParams.StorageBaseUrl, + s3StorageLocations[0].S3StorageLocationParams.StorageAwsRoleArn, + *s3StorageLocations[0].S3StorageLocationParams.StorageAwsExternalId, + s3StorageLocations[0].S3StorageLocationParams.Encryption.Type, + *s3StorageLocations[0].S3StorageLocationParams.Encryption.KmsKeyId, + ), + }, + ) + assert.Contains( + t, + trimmedProperties, + ExternalVolumePropNameValue{ + Name: "STORAGE_LOCATION_2", + Value: fmt.Sprintf( + `{"NAME":"%s","STORAGE_PROVIDER":"GCS","STORAGE_BASE_URL":"%s","ENCRYPTION_TYPE":"%s"}`, + gcsStorageLocations[0].GCSStorageLocationParams.Name, + gcsStorageLocations[0].GCSStorageLocationParams.StorageBaseUrl, + gcsStorageLocations[0].GCSStorageLocationParams.Encryption.Type, + ), + }, + ) + assert.Contains( + t, + trimmedProperties, + ExternalVolumePropNameValue{ + Name: "STORAGE_LOCATION_3", + Value: fmt.Sprintf( + `{"NAME":"%s","STORAGE_PROVIDER":"AZURE","STORAGE_BASE_URL":"%s","AZURE_TENANT_ID":"%s"}`, + azureStorageLocations[0].AzureStorageLocationParams.Name, + azureStorageLocations[0].AzureStorageLocationParams.StorageBaseUrl, + azureStorageLocations[0].AzureStorageLocationParams.AzureTenantId, + ), + }, + ) + assert.Contains( + t, + trimmedProperties, + ExternalVolumePropNameValue{ + Name: "STORAGE_LOCATION_4", + Value: fmt.Sprintf( + `{"NAME":"%s","STORAGE_PROVIDER":"%s","STORAGE_BASE_URL":"%s","STORAGE_AWS_ROLE_ARN":"%s","STORAGE_AWS_EXTERNAL_ID":"%s","ENCRYPTION_TYPE":"%s"}`, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.Name, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageProvider, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageBaseUrl, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageAwsRoleArn, + *s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageAwsExternalId, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.Encryption.Type, + ), + }, + ) }) - t.Run("Show with like", func(t *testing.T) { - allowWrites := true - comment := "some comment" + t.Run("Show with like", func(t *testing.T) { + allowWrites := true + comment := "some comment" id := createExternalVolume(t, s3StorageLocations, allowWrites, comment) - name := id.Name() - req := sdk.NewShowExternalVolumeRequest().WithLike(sdk.Like{Pattern: &name}) + name := id.Name() + req := sdk.NewShowExternalVolumeRequest().WithLike(sdk.Like{Pattern: &name}) externalVolumes, err := client.ExternalVolumes.Show(ctx, req) require.NoError(t, err) From cc129c71470355cff16995fb50d075086c28fc69 Mon Sep 17 00:00:00 2001 From: James Doldissen Date: Tue, 3 Sep 2024 00:00:33 +0200 Subject: [PATCH 03/10] fix: Finish TODOs --- .../external_volumes_gen_integration_test.go | 120 ++++++++++++------ 1 file changed, 79 insertions(+), 41 deletions(-) diff --git a/pkg/sdk/testint/external_volumes_gen_integration_test.go b/pkg/sdk/testint/external_volumes_gen_integration_test.go index 85ec0edc97..cd4161716b 100644 --- a/pkg/sdk/testint/external_volumes_gen_integration_test.go +++ b/pkg/sdk/testint/external_volumes_gen_integration_test.go @@ -13,8 +13,6 @@ import ( "github.com/stretchr/testify/require" ) -// Cannot test awsgov on non-gov Snowflake deployments - func TestInt_ExternalVolumes(t *testing.T) { client := testClient(t) ctx := testContext(t) @@ -25,9 +23,9 @@ func TestInt_ExternalVolumes(t *testing.T) { awsExternalId := "123456789" gcsBaseUrl := testenvs.GetOrSkipTest(t, testenvs.GcsExternalBuckerUrl) - // //TODO - To test external volumes wth gcs storage locations that have encryption TYPE=GCS_SSE_KMS a gcs KMS_KEY_ID is required, which is not currently available in testenvs + gcsKmsKeyId := "123456789" - azureBaseUrl := testenvs.GetOrSkipTest(t, testenvs.AzureExternalBucketUrl) // TODO verify assumption on form + azureBaseUrl := testenvs.GetOrSkipTest(t, testenvs.AzureExternalBucketUrl) azureTenantId := testenvs.GetOrSkipTest(t, testenvs.AzureExternalTenantId) assertExternalVolumeShowResult := func(t *testing.T, s *sdk.ExternalVolume, name sdk.AccountObjectIdentifier, allowWrites bool, comment string) { @@ -37,6 +35,7 @@ func TestInt_ExternalVolumes(t *testing.T) { assert.Equal(t, comment, s.Comment) } + // Structs for trimProperties function type ExternalVolumePropNameValue struct { Name string Value string @@ -168,6 +167,9 @@ func TestInt_ExternalVolumes(t *testing.T) { return externalVolumePropNameValue } + // Storage location structs for testing + // Note cannot test awsgov on non-gov Snowflake deployments + s3StorageLocations := []sdk.ExternalVolumeStorageLocation{ { S3StorageLocationParams: &sdk.S3StorageLocationParams{ @@ -184,7 +186,7 @@ func TestInt_ExternalVolumes(t *testing.T) { }, } - s3StorageLocationsNoneEncryption := []sdk.ExternalVolumeStorageLocation{ + s3StorageLocationsNoEncryption := []sdk.ExternalVolumeStorageLocation{ { S3StorageLocationParams: &sdk.S3StorageLocationParams{ Name: "s3_testing_storage_location_no_encryption", @@ -199,13 +201,26 @@ func TestInt_ExternalVolumes(t *testing.T) { }, } + gcsStorageLocationsNoEncryption := []sdk.ExternalVolumeStorageLocation{ + { + GCSStorageLocationParams: &sdk.GCSStorageLocationParams{ + Name: "gcs_testing_storage_location_no_encryption", + StorageBaseUrl: gcsBaseUrl, + Encryption: sdk.ExternalVolumeGCSEncryption{ + Type: sdk.GCSEncryptionTypeNone, + }, + }, + }, + } + gcsStorageLocations := []sdk.ExternalVolumeStorageLocation{ { GCSStorageLocationParams: &sdk.GCSStorageLocationParams{ Name: "gcs_testing_storage_location", StorageBaseUrl: gcsBaseUrl, Encryption: sdk.ExternalVolumeGCSEncryption{ - Type: sdk.GCSEncryptionTypeNone, + Type: sdk.GCSEncryptionTypeSseKms, + KmsKeyId: &gcsKmsKeyId, }, }, }, @@ -224,9 +239,7 @@ func TestInt_ExternalVolumes(t *testing.T) { createExternalVolume := func(t *testing.T, storageLocations []sdk.ExternalVolumeStorageLocation, allowWrites bool, comment string) sdk.AccountObjectIdentifier { t.Helper() - // id := testClientHelper().Ids.RandomAccountObjectIdentifier() - // TODO remove this - id := sdk.NewAccountObjectIdentifier("test_external_volume") + id := testClientHelper().Ids.RandomAccountObjectIdentifier() req := sdk.NewCreateExternalVolumeRequest(id, storageLocations). WithIfNotExists(true). WithAllowWrites(allowWrites). @@ -257,7 +270,7 @@ func TestInt_ExternalVolumes(t *testing.T) { t.Run("Create - S3 Storage Location No Encryption", func(t *testing.T) { allowWrites := true comment := "some comment" - id := createExternalVolume(t, s3StorageLocationsNoneEncryption, allowWrites, comment) + id := createExternalVolume(t, s3StorageLocationsNoEncryption, allowWrites, comment) externalVolume, err := client.ExternalVolumes.ShowByID(ctx, id) require.NoError(t, err) @@ -276,6 +289,17 @@ func TestInt_ExternalVolumes(t *testing.T) { assertExternalVolumeShowResult(t, externalVolume, id, allowWrites, comment) }) + t.Run("Create - GCS Storage Location No Encryption", func(t *testing.T) { + allowWrites := true + comment := "some comment" + id := createExternalVolume(t, gcsStorageLocationsNoEncryption, allowWrites, comment) + + externalVolume, err := client.ExternalVolumes.ShowByID(ctx, id) + require.NoError(t, err) + + assertExternalVolumeShowResult(t, externalVolume, id, allowWrites, comment) + }) + t.Run("Create - Azure Storage Location", func(t *testing.T) { allowWrites := true comment := "some comment" @@ -290,7 +314,7 @@ func TestInt_ExternalVolumes(t *testing.T) { t.Run("Create - Multiple Storage Locations", func(t *testing.T) { allowWrites := true comment := "some comment" - id := createExternalVolume(t, append(append(s3StorageLocations, gcsStorageLocations...), azureStorageLocations...), allowWrites, comment) + id := createExternalVolume(t, append(append(s3StorageLocations, gcsStorageLocationsNoEncryption...), azureStorageLocations...), allowWrites, comment) externalVolume, err := client.ExternalVolumes.ShowByID(ctx, id) require.NoError(t, err) @@ -301,9 +325,9 @@ func TestInt_ExternalVolumes(t *testing.T) { t.Run("Alter - remove storage location", func(t *testing.T) { allowWrites := true comment := "some comment" - id := createExternalVolume(t, append(s3StorageLocationsNoneEncryption, gcsStorageLocations...), allowWrites, comment) + id := createExternalVolume(t, append(s3StorageLocationsNoEncryption, gcsStorageLocationsNoEncryption...), allowWrites, comment) - req := sdk.NewAlterExternalVolumeRequest(id).WithRemoveStorageLocation(gcsStorageLocations[0].GCSStorageLocationParams.Name) + req := sdk.NewAlterExternalVolumeRequest(id).WithRemoveStorageLocation(gcsStorageLocationsNoEncryption[0].GCSStorageLocationParams.Name) err := client.ExternalVolumes.Alter(ctx, req) require.NoError(t, err) @@ -323,12 +347,12 @@ func TestInt_ExternalVolumes(t *testing.T) { Name: "STORAGE_LOCATION_1", Value: fmt.Sprintf( `{"NAME":"%s","STORAGE_PROVIDER":"%s","STORAGE_BASE_URL":"%s","STORAGE_AWS_ROLE_ARN":"%s","STORAGE_AWS_EXTERNAL_ID":"%s","ENCRYPTION_TYPE":"%s"}`, - s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.Name, - s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageProvider, - s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageBaseUrl, - s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageAwsRoleArn, - *s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageAwsExternalId, - s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.Encryption.Type, + s3StorageLocationsNoEncryption[0].S3StorageLocationParams.Name, + s3StorageLocationsNoEncryption[0].S3StorageLocationParams.StorageProvider, + s3StorageLocationsNoEncryption[0].S3StorageLocationParams.StorageBaseUrl, + s3StorageLocationsNoEncryption[0].S3StorageLocationParams.StorageAwsRoleArn, + *s3StorageLocationsNoEncryption[0].S3StorageLocationParams.StorageAwsExternalId, + s3StorageLocationsNoEncryption[0].S3StorageLocationParams.Encryption.Type, ), }, ) @@ -337,7 +361,7 @@ func TestInt_ExternalVolumes(t *testing.T) { t.Run("Alter - set comment", func(t *testing.T) { allowWrites := true comment := "" - id := createExternalVolume(t, s3StorageLocationsNoneEncryption, allowWrites, "some comment") + id := createExternalVolume(t, s3StorageLocationsNoEncryption, allowWrites, "some comment") req := sdk.NewAlterExternalVolumeRequest(id).WithSet( *sdk.NewAlterExternalVolumeSetRequest().WithComment(comment), @@ -360,12 +384,12 @@ func TestInt_ExternalVolumes(t *testing.T) { Name: "STORAGE_LOCATION_1", Value: fmt.Sprintf( `{"NAME":"%s","STORAGE_PROVIDER":"%s","STORAGE_BASE_URL":"%s","STORAGE_AWS_ROLE_ARN":"%s","STORAGE_AWS_EXTERNAL_ID":"%s","ENCRYPTION_TYPE":"%s"}`, - s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.Name, - s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageProvider, - s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageBaseUrl, - s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageAwsRoleArn, - *s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageAwsExternalId, - s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.Encryption.Type, + s3StorageLocationsNoEncryption[0].S3StorageLocationParams.Name, + s3StorageLocationsNoEncryption[0].S3StorageLocationParams.StorageProvider, + s3StorageLocationsNoEncryption[0].S3StorageLocationParams.StorageBaseUrl, + s3StorageLocationsNoEncryption[0].S3StorageLocationParams.StorageAwsRoleArn, + *s3StorageLocationsNoEncryption[0].S3StorageLocationParams.StorageAwsExternalId, + s3StorageLocationsNoEncryption[0].S3StorageLocationParams.Encryption.Type, ), }, ) @@ -413,7 +437,7 @@ func TestInt_ExternalVolumes(t *testing.T) { t.Run("Alter - add s3 storage location to external volume", func(t *testing.T) { allowWrites := true comment := "some comment" - id := createExternalVolume(t, gcsStorageLocations, allowWrites, comment) + id := createExternalVolume(t, gcsStorageLocationsNoEncryption, allowWrites, comment) req := sdk.NewAlterExternalVolumeRequest(id).WithAddStorageLocation( *sdk.NewExternalVolumeStorageLocationRequest().WithS3StorageLocationParams( @@ -447,9 +471,9 @@ func TestInt_ExternalVolumes(t *testing.T) { Name: "STORAGE_LOCATION_1", Value: fmt.Sprintf( `{"NAME":"%s","STORAGE_PROVIDER":"GCS","STORAGE_BASE_URL":"%s","ENCRYPTION_TYPE":"%s"}`, - gcsStorageLocations[0].GCSStorageLocationParams.Name, - gcsStorageLocations[0].GCSStorageLocationParams.StorageBaseUrl, - gcsStorageLocations[0].GCSStorageLocationParams.Encryption.Type, + gcsStorageLocationsNoEncryption[0].GCSStorageLocationParams.Name, + gcsStorageLocationsNoEncryption[0].GCSStorageLocationParams.StorageBaseUrl, + gcsStorageLocationsNoEncryption[0].GCSStorageLocationParams.Encryption.Type, ), }, ) @@ -477,7 +501,7 @@ func TestInt_ExternalVolumes(t *testing.T) { comment := "some comment" id := createExternalVolume( t, - append(append(append(s3StorageLocations, gcsStorageLocations...), azureStorageLocations...), s3StorageLocationsNoneEncryption...), + append(append(append(append(s3StorageLocations, gcsStorageLocationsNoEncryption...), azureStorageLocations...), s3StorageLocationsNoEncryption...), gcsStorageLocations...), allowWrites, comment, ) @@ -486,7 +510,7 @@ func TestInt_ExternalVolumes(t *testing.T) { require.NoError(t, err) trimmedProperties := trimProperties(t, props) - assert.Equal(t, 7, len(trimmedProperties)) + assert.Equal(t, 8, len(trimmedProperties)) assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "COMMENT", Value: comment}) assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "ALLOW_WRITES", Value: strconv.FormatBool(allowWrites)}) assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "ACTIVE", Value: ""}) @@ -514,9 +538,9 @@ func TestInt_ExternalVolumes(t *testing.T) { Name: "STORAGE_LOCATION_2", Value: fmt.Sprintf( `{"NAME":"%s","STORAGE_PROVIDER":"GCS","STORAGE_BASE_URL":"%s","ENCRYPTION_TYPE":"%s"}`, - gcsStorageLocations[0].GCSStorageLocationParams.Name, - gcsStorageLocations[0].GCSStorageLocationParams.StorageBaseUrl, - gcsStorageLocations[0].GCSStorageLocationParams.Encryption.Type, + gcsStorageLocationsNoEncryption[0].GCSStorageLocationParams.Name, + gcsStorageLocationsNoEncryption[0].GCSStorageLocationParams.StorageBaseUrl, + gcsStorageLocationsNoEncryption[0].GCSStorageLocationParams.Encryption.Type, ), }, ) @@ -540,12 +564,26 @@ func TestInt_ExternalVolumes(t *testing.T) { Name: "STORAGE_LOCATION_4", Value: fmt.Sprintf( `{"NAME":"%s","STORAGE_PROVIDER":"%s","STORAGE_BASE_URL":"%s","STORAGE_AWS_ROLE_ARN":"%s","STORAGE_AWS_EXTERNAL_ID":"%s","ENCRYPTION_TYPE":"%s"}`, - s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.Name, - s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageProvider, - s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageBaseUrl, - s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageAwsRoleArn, - *s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageAwsExternalId, - s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.Encryption.Type, + s3StorageLocationsNoEncryption[0].S3StorageLocationParams.Name, + s3StorageLocationsNoEncryption[0].S3StorageLocationParams.StorageProvider, + s3StorageLocationsNoEncryption[0].S3StorageLocationParams.StorageBaseUrl, + s3StorageLocationsNoEncryption[0].S3StorageLocationParams.StorageAwsRoleArn, + *s3StorageLocationsNoEncryption[0].S3StorageLocationParams.StorageAwsExternalId, + s3StorageLocationsNoEncryption[0].S3StorageLocationParams.Encryption.Type, + ), + }, + ) + assert.Contains( + t, + trimmedProperties, + ExternalVolumePropNameValue{ + Name: "STORAGE_LOCATION_5", + Value: fmt.Sprintf( + `{"NAME":"%s","STORAGE_PROVIDER":"GCS","STORAGE_BASE_URL":"%s","ENCRYPTION_TYPE":"%s","ENCRYPTION_KMS_KEY_ID":"%s"}`, + gcsStorageLocations[0].GCSStorageLocationParams.Name, + gcsStorageLocations[0].GCSStorageLocationParams.StorageBaseUrl, + gcsStorageLocations[0].GCSStorageLocationParams.Encryption.Type, + *gcsStorageLocations[0].GCSStorageLocationParams.Encryption.KmsKeyId, ), }, ) From 4821b1bd3c4067536f87e933b64be0a0df5a1110 Mon Sep 17 00:00:00 2001 From: James Doldissen Date: Tue, 3 Sep 2024 19:20:54 +0200 Subject: [PATCH 04/10] fix: Make encryption optional for s3 and gcs storage locations --- pkg/sdk/external_volumes_def.go | 8 +- pkg/sdk/external_volumes_dto_builders_gen.go | 14 +- pkg/sdk/external_volumes_dto_gen.go | 8 +- pkg/sdk/external_volumes_gen.go | 20 +-- pkg/sdk/external_volumes_gen_test.go | 69 ++++++-- pkg/sdk/external_volumes_impl_gen.go | 24 ++- .../external_volumes_gen_integration_test.go | 165 +++++++++++++----- 7 files changed, 224 insertions(+), 84 deletions(-) diff --git a/pkg/sdk/external_volumes_def.go b/pkg/sdk/external_volumes_def.go index 01c7d00174..56abe176fa 100644 --- a/pkg/sdk/external_volumes_def.go +++ b/pkg/sdk/external_volumes_def.go @@ -26,24 +26,24 @@ var externalS3StorageLocationDef = g.NewQueryStruct("S3StorageLocationParams"). TextAssignment("STORAGE_AWS_ROLE_ARN", g.ParameterOptions().SingleQuotes().Required()). TextAssignment("STORAGE_BASE_URL", g.ParameterOptions().SingleQuotes().Required()). OptionalTextAssignment("STORAGE_AWS_EXTERNAL_ID", g.ParameterOptions().SingleQuotes()). - QueryStructField( + OptionalQueryStructField( "Encryption", g.NewQueryStruct("ExternalVolumeS3Encryption"). Assignment("TYPE", g.KindOfT[S3EncryptionType](), g.ParameterOptions().SingleQuotes().Required()). OptionalTextAssignment("KMS_KEY_ID", g.ParameterOptions().SingleQuotes()), - g.ListOptions().Parentheses().NoComma().SQL("ENCRYPTION =").Required(), + g.ListOptions().Parentheses().NoComma().SQL("ENCRYPTION ="), ) var externalGCSStorageLocationDef = g.NewQueryStruct("GCSStorageLocationParams"). TextAssignment("NAME", g.ParameterOptions().SingleQuotes().Required()). PredefinedQueryStructField("StorageProviderGcs", "string", g.StaticOptions().SQL("STORAGE_PROVIDER = 'GCS'")). TextAssignment("STORAGE_BASE_URL", g.ParameterOptions().SingleQuotes().Required()). - QueryStructField( + OptionalQueryStructField( "Encryption", g.NewQueryStruct("ExternalVolumeGCSEncryption"). Assignment("TYPE", g.KindOfT[GCSEncryptionType](), g.ParameterOptions().SingleQuotes().Required()). OptionalTextAssignment("KMS_KEY_ID", g.ParameterOptions().SingleQuotes()), - g.ListOptions().Parentheses().NoComma().SQL("ENCRYPTION =").Required(), + g.ListOptions().Parentheses().NoComma().SQL("ENCRYPTION ="), ) var externalAzureStorageLocationDef = g.NewQueryStruct("AzureStorageLocationParams"). diff --git a/pkg/sdk/external_volumes_dto_builders_gen.go b/pkg/sdk/external_volumes_dto_builders_gen.go index abb1ba98d1..3dc42ef7c8 100644 --- a/pkg/sdk/external_volumes_dto_builders_gen.go +++ b/pkg/sdk/external_volumes_dto_builders_gen.go @@ -100,14 +100,12 @@ func NewS3StorageLocationParamsRequest( StorageProvider S3StorageProvider, StorageAwsRoleArn string, StorageBaseUrl string, - Encryption ExternalVolumeS3EncryptionRequest, ) *S3StorageLocationParamsRequest { s := S3StorageLocationParamsRequest{} s.Name = Name s.StorageProvider = StorageProvider s.StorageAwsRoleArn = StorageAwsRoleArn s.StorageBaseUrl = StorageBaseUrl - s.Encryption = Encryption return &s } @@ -116,6 +114,11 @@ func (s *S3StorageLocationParamsRequest) WithStorageAwsExternalId(StorageAwsExte return s } +func (s *S3StorageLocationParamsRequest) WithEncryption(Encryption ExternalVolumeS3EncryptionRequest) *S3StorageLocationParamsRequest { + s.Encryption = &Encryption + return s +} + func NewExternalVolumeS3EncryptionRequest( Type S3EncryptionType, ) *ExternalVolumeS3EncryptionRequest { @@ -132,15 +135,18 @@ func (s *ExternalVolumeS3EncryptionRequest) WithKmsKeyId(KmsKeyId string) *Exter func NewGCSStorageLocationParamsRequest( Name string, StorageBaseUrl string, - Encryption ExternalVolumeGCSEncryptionRequest, ) *GCSStorageLocationParamsRequest { s := GCSStorageLocationParamsRequest{} s.Name = Name s.StorageBaseUrl = StorageBaseUrl - s.Encryption = Encryption return &s } +func (s *GCSStorageLocationParamsRequest) WithEncryption(Encryption ExternalVolumeGCSEncryptionRequest) *GCSStorageLocationParamsRequest { + s.Encryption = &Encryption + return s +} + func NewExternalVolumeGCSEncryptionRequest( Type GCSEncryptionType, ) *ExternalVolumeGCSEncryptionRequest { diff --git a/pkg/sdk/external_volumes_dto_gen.go b/pkg/sdk/external_volumes_dto_gen.go index ec01cb5603..a62c329f60 100644 --- a/pkg/sdk/external_volumes_dto_gen.go +++ b/pkg/sdk/external_volumes_dto_gen.go @@ -44,7 +44,7 @@ type S3StorageLocationParamsRequest struct { StorageAwsRoleArn string // required StorageBaseUrl string // required StorageAwsExternalId *string - Encryption ExternalVolumeS3EncryptionRequest // required + Encryption *ExternalVolumeS3EncryptionRequest } type ExternalVolumeS3EncryptionRequest struct { @@ -53,9 +53,9 @@ type ExternalVolumeS3EncryptionRequest struct { } type GCSStorageLocationParamsRequest struct { - Name string // required - StorageBaseUrl string // required - Encryption ExternalVolumeGCSEncryptionRequest // required + Name string // required + StorageBaseUrl string // required + Encryption *ExternalVolumeGCSEncryptionRequest } type ExternalVolumeGCSEncryptionRequest struct { diff --git a/pkg/sdk/external_volumes_gen.go b/pkg/sdk/external_volumes_gen.go index 659163da97..726e3ab819 100644 --- a/pkg/sdk/external_volumes_gen.go +++ b/pkg/sdk/external_volumes_gen.go @@ -28,22 +28,22 @@ type ExternalVolumeStorageLocation struct { AzureStorageLocationParams *AzureStorageLocationParams `ddl:"list,parentheses,no_comma"` } type S3StorageLocationParams struct { - Name string `ddl:"parameter,single_quotes" sql:"NAME"` - StorageProvider S3StorageProvider `ddl:"parameter,single_quotes" sql:"STORAGE_PROVIDER"` - StorageAwsRoleArn string `ddl:"parameter,single_quotes" sql:"STORAGE_AWS_ROLE_ARN"` - StorageBaseUrl string `ddl:"parameter,single_quotes" sql:"STORAGE_BASE_URL"` - StorageAwsExternalId *string `ddl:"parameter,single_quotes" sql:"STORAGE_AWS_EXTERNAL_ID"` - Encryption ExternalVolumeS3Encryption `ddl:"list,parentheses,no_comma" sql:"ENCRYPTION ="` + Name string `ddl:"parameter,single_quotes" sql:"NAME"` + StorageProvider S3StorageProvider `ddl:"parameter,single_quotes" sql:"STORAGE_PROVIDER"` + StorageAwsRoleArn string `ddl:"parameter,single_quotes" sql:"STORAGE_AWS_ROLE_ARN"` + StorageBaseUrl string `ddl:"parameter,single_quotes" sql:"STORAGE_BASE_URL"` + StorageAwsExternalId *string `ddl:"parameter,single_quotes" sql:"STORAGE_AWS_EXTERNAL_ID"` + Encryption *ExternalVolumeS3Encryption `ddl:"list,parentheses,no_comma" sql:"ENCRYPTION ="` } type ExternalVolumeS3Encryption struct { Type S3EncryptionType `ddl:"parameter,single_quotes" sql:"TYPE"` KmsKeyId *string `ddl:"parameter,single_quotes" sql:"KMS_KEY_ID"` } type GCSStorageLocationParams struct { - Name string `ddl:"parameter,single_quotes" sql:"NAME"` - StorageProviderGcs string `ddl:"static" sql:"STORAGE_PROVIDER = 'GCS'"` - StorageBaseUrl string `ddl:"parameter,single_quotes" sql:"STORAGE_BASE_URL"` - Encryption ExternalVolumeGCSEncryption `ddl:"list,parentheses,no_comma" sql:"ENCRYPTION ="` + Name string `ddl:"parameter,single_quotes" sql:"NAME"` + StorageProviderGcs string `ddl:"static" sql:"STORAGE_PROVIDER = 'GCS'"` + StorageBaseUrl string `ddl:"parameter,single_quotes" sql:"STORAGE_BASE_URL"` + Encryption *ExternalVolumeGCSEncryption `ddl:"list,parentheses,no_comma" sql:"ENCRYPTION ="` } type ExternalVolumeGCSEncryption struct { Type GCSEncryptionType `ddl:"parameter,single_quotes" sql:"TYPE"` diff --git a/pkg/sdk/external_volumes_gen_test.go b/pkg/sdk/external_volumes_gen_test.go index 15c3118ceb..e4b20dbc6a 100644 --- a/pkg/sdk/external_volumes_gen_test.go +++ b/pkg/sdk/external_volumes_gen_test.go @@ -8,28 +8,35 @@ var s3StorageLocationParams = &S3StorageLocationParams{ StorageProvider: S3StorageProviderS3, StorageAwsRoleArn: "some s3 role arn", StorageBaseUrl: "some s3 base url", - Encryption: ExternalVolumeS3Encryption{ + Encryption: &ExternalVolumeS3Encryption{ Type: S3EncryptionTypeSseS3, KmsKeyId: String("some s3 kms key id"), }, } -var s3StorageLocationParamsNoEncryption = &S3StorageLocationParams{ +var s3StorageLocationParamsNoneEncryption = &S3StorageLocationParams{ Name: "some s3 name", StorageProvider: S3StorageProviderS3, StorageAwsRoleArn: "some s3 role arn", StorageBaseUrl: "some s3 base url", - Encryption: ExternalVolumeS3Encryption{ + Encryption: &ExternalVolumeS3Encryption{ Type: S3EncryptionNone, }, } +var s3StorageLocationParamsNoEncryption = &S3StorageLocationParams{ + Name: "some s3 name", + StorageProvider: S3StorageProviderS3, + StorageAwsRoleArn: "some s3 role arn", + StorageBaseUrl: "some s3 base url", +} + var s3StorageLocationParamsGov = &S3StorageLocationParams{ Name: "some s3 name", StorageProvider: S3StorageProviderS3GOV, StorageAwsRoleArn: "some s3 role arn", StorageBaseUrl: "some s3 base url", - Encryption: ExternalVolumeS3Encryption{ + Encryption: &ExternalVolumeS3Encryption{ Type: S3EncryptionTypeSseS3, KmsKeyId: String("some s3 kms key id"), }, @@ -41,7 +48,7 @@ var s3StorageLocationParamsWithExternalId = &S3StorageLocationParams{ StorageAwsRoleArn: "some s3 role arn", StorageBaseUrl: "some s3 base url", StorageAwsExternalId: String("some s3 external id"), - Encryption: ExternalVolumeS3Encryption{ + Encryption: &ExternalVolumeS3Encryption{ Type: S3EncryptionTypeSseS3, KmsKeyId: String("some s3 kms key id"), }, @@ -50,20 +57,25 @@ var s3StorageLocationParamsWithExternalId = &S3StorageLocationParams{ var gcsStorageLocationParams = &GCSStorageLocationParams{ Name: "some gcs name", StorageBaseUrl: "some gcs base url", - Encryption: ExternalVolumeGCSEncryption{ + Encryption: &ExternalVolumeGCSEncryption{ Type: GCSEncryptionTypeSseKms, KmsKeyId: String("some gcs kms key id"), }, } -var gcsStorageLocationParamsNoEncryption = &GCSStorageLocationParams{ +var gcsStorageLocationParamsNoneEncryption = &GCSStorageLocationParams{ Name: "some gcs name", StorageBaseUrl: "some gcs base url", - Encryption: ExternalVolumeGCSEncryption{ + Encryption: &ExternalVolumeGCSEncryption{ Type: GCSEncryptionTypeNone, }, } +var gcsStorageLocationParamsNoEncryption = &GCSStorageLocationParams{ + Name: "some gcs name", + StorageBaseUrl: "some gcs base url", +} + var azureStorageLocationParams = &AzureStorageLocationParams{ Name: "some azure name", AzureTenantId: "some azure tenant id", @@ -141,10 +153,16 @@ func TestExternalVolumes_Create(t *testing.T) { assertOptsValidAndSQLEquals(t, opts, `CREATE EXTERNAL VOLUME %s STORAGE_LOCATIONS = ((NAME = 'some s3 name' STORAGE_PROVIDER = 'S3GOV' STORAGE_AWS_ROLE_ARN = 'some s3 role arn' STORAGE_BASE_URL = 'some s3 base url' ENCRYPTION = (TYPE = 'AWS_SSE_S3' KMS_KEY_ID = 'some s3 kms key id'))) COMMENT = 'some comment'`, id.FullyQualifiedName()) }) + t.Run("1 storage location - s3 none encryption", func(t *testing.T) { + opts := defaultOpts() + opts.StorageLocations = []ExternalVolumeStorageLocation{{S3StorageLocationParams: s3StorageLocationParamsNoneEncryption}} + assertOptsValidAndSQLEquals(t, opts, `CREATE EXTERNAL VOLUME %s STORAGE_LOCATIONS = ((NAME = 'some s3 name' STORAGE_PROVIDER = 'S3' STORAGE_AWS_ROLE_ARN = 'some s3 role arn' STORAGE_BASE_URL = 'some s3 base url' ENCRYPTION = (TYPE = 'NONE')))`, id.FullyQualifiedName()) + }) + t.Run("1 storage location - s3 no encryption", func(t *testing.T) { opts := defaultOpts() opts.StorageLocations = []ExternalVolumeStorageLocation{{S3StorageLocationParams: s3StorageLocationParamsNoEncryption}} - assertOptsValidAndSQLEquals(t, opts, `CREATE EXTERNAL VOLUME %s STORAGE_LOCATIONS = ((NAME = 'some s3 name' STORAGE_PROVIDER = 'S3' STORAGE_AWS_ROLE_ARN = 'some s3 role arn' STORAGE_BASE_URL = 'some s3 base url' ENCRYPTION = (TYPE = 'NONE')))`, id.FullyQualifiedName()) + assertOptsValidAndSQLEquals(t, opts, `CREATE EXTERNAL VOLUME %s STORAGE_LOCATIONS = ((NAME = 'some s3 name' STORAGE_PROVIDER = 'S3' STORAGE_AWS_ROLE_ARN = 'some s3 role arn' STORAGE_BASE_URL = 'some s3 base url'))`, id.FullyQualifiedName()) }) t.Run("1 storage location with allow writes - gcs", func(t *testing.T) { @@ -154,11 +172,18 @@ func TestExternalVolumes_Create(t *testing.T) { assertOptsValidAndSQLEquals(t, opts, `CREATE EXTERNAL VOLUME %s STORAGE_LOCATIONS = ((NAME = 'some gcs name' STORAGE_PROVIDER = 'GCS' STORAGE_BASE_URL = 'some gcs base url' ENCRYPTION = (TYPE = 'GCS_SSE_KMS' KMS_KEY_ID = 'some gcs kms key id'))) ALLOW_WRITES = true`, id.FullyQualifiedName()) }) + t.Run("1 storage location with allow writes - gcs none encryption", func(t *testing.T) { + opts := defaultOpts() + opts.StorageLocations = []ExternalVolumeStorageLocation{{GCSStorageLocationParams: gcsStorageLocationParamsNoneEncryption}} + opts.AllowWrites = Bool(true) + assertOptsValidAndSQLEquals(t, opts, `CREATE EXTERNAL VOLUME %s STORAGE_LOCATIONS = ((NAME = 'some gcs name' STORAGE_PROVIDER = 'GCS' STORAGE_BASE_URL = 'some gcs base url' ENCRYPTION = (TYPE = 'NONE'))) ALLOW_WRITES = true`, id.FullyQualifiedName()) + }) + t.Run("1 storage location with allow writes - gcs no encryption", func(t *testing.T) { opts := defaultOpts() opts.StorageLocations = []ExternalVolumeStorageLocation{{GCSStorageLocationParams: gcsStorageLocationParamsNoEncryption}} opts.AllowWrites = Bool(true) - assertOptsValidAndSQLEquals(t, opts, `CREATE EXTERNAL VOLUME %s STORAGE_LOCATIONS = ((NAME = 'some gcs name' STORAGE_PROVIDER = 'GCS' STORAGE_BASE_URL = 'some gcs base url' ENCRYPTION = (TYPE = 'NONE'))) ALLOW_WRITES = true`, id.FullyQualifiedName()) + assertOptsValidAndSQLEquals(t, opts, `CREATE EXTERNAL VOLUME %s STORAGE_LOCATIONS = ((NAME = 'some gcs name' STORAGE_PROVIDER = 'GCS' STORAGE_BASE_URL = 'some gcs base url')) ALLOW_WRITES = true`, id.FullyQualifiedName()) }) t.Run("1 storage location - azure", func(t *testing.T) { @@ -289,12 +314,36 @@ func TestExternalVolumes_Alter(t *testing.T) { assertOptsValidAndSQLEquals(t, opts, `ALTER EXTERNAL VOLUME %s ADD STORAGE_LOCATION = (NAME = 'some s3 name' STORAGE_PROVIDER = 'S3' STORAGE_AWS_ROLE_ARN = 'some s3 role arn' STORAGE_BASE_URL = 'some s3 base url' STORAGE_AWS_EXTERNAL_ID = 'some s3 external id' ENCRYPTION = (TYPE = 'AWS_SSE_S3' KMS_KEY_ID = 'some s3 kms key id'))`, id.FullyQualifiedName()) }) + t.Run("add storage location - s3 none encryption", func(t *testing.T) { + opts := defaultOpts() + opts.AddStorageLocation = &ExternalVolumeStorageLocation{S3StorageLocationParams: s3StorageLocationParamsNoneEncryption} + assertOptsValidAndSQLEquals(t, opts, `ALTER EXTERNAL VOLUME %s ADD STORAGE_LOCATION = (NAME = 'some s3 name' STORAGE_PROVIDER = 'S3' STORAGE_AWS_ROLE_ARN = 'some s3 role arn' STORAGE_BASE_URL = 'some s3 base url' ENCRYPTION = (TYPE = 'NONE'))`, id.FullyQualifiedName()) + }) + + t.Run("add storage location - s3 no encryption", func(t *testing.T) { + opts := defaultOpts() + opts.AddStorageLocation = &ExternalVolumeStorageLocation{S3StorageLocationParams: s3StorageLocationParamsNoEncryption} + assertOptsValidAndSQLEquals(t, opts, `ALTER EXTERNAL VOLUME %s ADD STORAGE_LOCATION = (NAME = 'some s3 name' STORAGE_PROVIDER = 'S3' STORAGE_AWS_ROLE_ARN = 'some s3 role arn' STORAGE_BASE_URL = 'some s3 base url')`, id.FullyQualifiedName()) + }) + t.Run("add storage location - gcs", func(t *testing.T) { opts := defaultOpts() opts.AddStorageLocation = &ExternalVolumeStorageLocation{GCSStorageLocationParams: gcsStorageLocationParams} assertOptsValidAndSQLEquals(t, opts, `ALTER EXTERNAL VOLUME %s ADD STORAGE_LOCATION = (NAME = 'some gcs name' STORAGE_PROVIDER = 'GCS' STORAGE_BASE_URL = 'some gcs base url' ENCRYPTION = (TYPE = 'GCS_SSE_KMS' KMS_KEY_ID = 'some gcs kms key id'))`, id.FullyQualifiedName()) }) + t.Run("add storage location - gcs none encryption", func(t *testing.T) { + opts := defaultOpts() + opts.AddStorageLocation = &ExternalVolumeStorageLocation{GCSStorageLocationParams: gcsStorageLocationParamsNoneEncryption} + assertOptsValidAndSQLEquals(t, opts, `ALTER EXTERNAL VOLUME %s ADD STORAGE_LOCATION = (NAME = 'some gcs name' STORAGE_PROVIDER = 'GCS' STORAGE_BASE_URL = 'some gcs base url' ENCRYPTION = (TYPE = 'NONE'))`, id.FullyQualifiedName()) + }) + + t.Run("add storage location - gcs no encryption", func(t *testing.T) { + opts := defaultOpts() + opts.AddStorageLocation = &ExternalVolumeStorageLocation{GCSStorageLocationParams: gcsStorageLocationParamsNoEncryption} + assertOptsValidAndSQLEquals(t, opts, `ALTER EXTERNAL VOLUME %s ADD STORAGE_LOCATION = (NAME = 'some gcs name' STORAGE_PROVIDER = 'GCS' STORAGE_BASE_URL = 'some gcs base url')`, id.FullyQualifiedName()) + }) + t.Run("add storage location - azure", func(t *testing.T) { opts := defaultOpts() opts.AddStorageLocation = &ExternalVolumeStorageLocation{AzureStorageLocationParams: azureStorageLocationParams} diff --git a/pkg/sdk/external_volumes_impl_gen.go b/pkg/sdk/external_volumes_impl_gen.go index 004f53e10c..356f2122bf 100644 --- a/pkg/sdk/external_volumes_impl_gen.go +++ b/pkg/sdk/external_volumes_impl_gen.go @@ -76,10 +76,12 @@ func (r *AlterExternalVolumeRequest) toOpts() *AlterExternalVolumeOptions { } if r.Set != nil { + opts.Set = &AlterExternalVolumeSet{ AllowWrites: r.Set.AllowWrites, Comment: r.Set.Comment, } + } if r.AddStorageLocation != nil { @@ -96,9 +98,13 @@ func (r *AlterExternalVolumeRequest) toOpts() *AlterExternalVolumeOptions { StorageAwsExternalId: r.AddStorageLocation.S3StorageLocationParams.StorageAwsExternalId, } - opts.AddStorageLocation.S3StorageLocationParams.Encryption = ExternalVolumeS3Encryption{ - Type: r.AddStorageLocation.S3StorageLocationParams.Encryption.Type, - KmsKeyId: r.AddStorageLocation.S3StorageLocationParams.Encryption.KmsKeyId, + if r.AddStorageLocation.S3StorageLocationParams.Encryption != nil { + + opts.AddStorageLocation.S3StorageLocationParams.Encryption = &ExternalVolumeS3Encryption{ + Type: r.AddStorageLocation.S3StorageLocationParams.Encryption.Type, + KmsKeyId: r.AddStorageLocation.S3StorageLocationParams.Encryption.KmsKeyId, + } + } } @@ -110,19 +116,25 @@ func (r *AlterExternalVolumeRequest) toOpts() *AlterExternalVolumeOptions { StorageBaseUrl: r.AddStorageLocation.GCSStorageLocationParams.StorageBaseUrl, } - opts.AddStorageLocation.GCSStorageLocationParams.Encryption = ExternalVolumeGCSEncryption{ - Type: r.AddStorageLocation.GCSStorageLocationParams.Encryption.Type, - KmsKeyId: r.AddStorageLocation.GCSStorageLocationParams.Encryption.KmsKeyId, + if r.AddStorageLocation.GCSStorageLocationParams.Encryption != nil { + + opts.AddStorageLocation.GCSStorageLocationParams.Encryption = &ExternalVolumeGCSEncryption{ + Type: r.AddStorageLocation.GCSStorageLocationParams.Encryption.Type, + KmsKeyId: r.AddStorageLocation.GCSStorageLocationParams.Encryption.KmsKeyId, + } + } } if r.AddStorageLocation.AzureStorageLocationParams != nil { + opts.AddStorageLocation.AzureStorageLocationParams = &AzureStorageLocationParams{ Name: r.AddStorageLocation.AzureStorageLocationParams.Name, AzureTenantId: r.AddStorageLocation.AzureStorageLocationParams.AzureTenantId, StorageBaseUrl: r.AddStorageLocation.AzureStorageLocationParams.StorageBaseUrl, } + } } diff --git a/pkg/sdk/testint/external_volumes_gen_integration_test.go b/pkg/sdk/testint/external_volumes_gen_integration_test.go index cd4161716b..ff945f13e6 100644 --- a/pkg/sdk/testint/external_volumes_gen_integration_test.go +++ b/pkg/sdk/testint/external_volumes_gen_integration_test.go @@ -59,7 +59,7 @@ func TestInt_ExternalVolumes(t *testing.T) { StorageBaseUrl string `json:"STORAGE_BASE_URL"` StorageAwsRoleArn string `json:"STORAGE_AWS_ROLE_ARN"` StorageAwsExternalId string `json:"STORAGE_AWS_EXTERNAL_ID"` - EncryptionType string `json:"ENCRYPTION_TYPE"` + EncryptionType string `json:"ENCRYPTION_TYPE,omitempty"` EncryptionKmsId string `json:"ENCRYPTION_KMS_KEY_ID,omitempty"` } @@ -77,7 +77,7 @@ func TestInt_ExternalVolumes(t *testing.T) { Name string `json:"NAME"` StorageProvider string `json:"STORAGE_PROVIDER"` StorageBaseUrl string `json:"STORAGE_BASE_URL"` - EncryptionType string `json:"ENCRYPTION_TYPE"` + EncryptionType string `json:"ENCRYPTION_TYPE,omitempty"` EncryptionKmsId string `json:"ENCRYPTION_KMS_KEY_ID,omitempty"` } @@ -178,7 +178,7 @@ func TestInt_ExternalVolumes(t *testing.T) { StorageAwsRoleArn: awsRoleARN, StorageBaseUrl: awsBaseUrl, StorageAwsExternalId: sdk.String(awsExternalId), - Encryption: sdk.ExternalVolumeS3Encryption{ + Encryption: &sdk.ExternalVolumeS3Encryption{ Type: sdk.S3EncryptionTypeSseKms, KmsKeyId: &awsKmsKeyId, }, @@ -186,6 +186,21 @@ func TestInt_ExternalVolumes(t *testing.T) { }, } + s3StorageLocationsNoneEncryption := []sdk.ExternalVolumeStorageLocation{ + { + S3StorageLocationParams: &sdk.S3StorageLocationParams{ + Name: "s3_testing_storage_location_none_encryption", + StorageProvider: sdk.S3StorageProviderS3, + StorageAwsRoleArn: awsRoleARN, + StorageBaseUrl: awsBaseUrl, + StorageAwsExternalId: sdk.String(awsExternalId), + Encryption: &sdk.ExternalVolumeS3Encryption{ + Type: sdk.S3EncryptionNone, + }, + }, + }, + } + s3StorageLocationsNoEncryption := []sdk.ExternalVolumeStorageLocation{ { S3StorageLocationParams: &sdk.S3StorageLocationParams{ @@ -194,8 +209,17 @@ func TestInt_ExternalVolumes(t *testing.T) { StorageAwsRoleArn: awsRoleARN, StorageBaseUrl: awsBaseUrl, StorageAwsExternalId: sdk.String(awsExternalId), - Encryption: sdk.ExternalVolumeS3Encryption{ - Type: sdk.S3EncryptionNone, + }, + }, + } + + gcsStorageLocationsNoneEncryption := []sdk.ExternalVolumeStorageLocation{ + { + GCSStorageLocationParams: &sdk.GCSStorageLocationParams{ + Name: "gcs_testing_storage_location_none_encryption", + StorageBaseUrl: gcsBaseUrl, + Encryption: &sdk.ExternalVolumeGCSEncryption{ + Type: sdk.GCSEncryptionTypeNone, }, }, }, @@ -206,9 +230,6 @@ func TestInt_ExternalVolumes(t *testing.T) { GCSStorageLocationParams: &sdk.GCSStorageLocationParams{ Name: "gcs_testing_storage_location_no_encryption", StorageBaseUrl: gcsBaseUrl, - Encryption: sdk.ExternalVolumeGCSEncryption{ - Type: sdk.GCSEncryptionTypeNone, - }, }, }, } @@ -218,7 +239,7 @@ func TestInt_ExternalVolumes(t *testing.T) { GCSStorageLocationParams: &sdk.GCSStorageLocationParams{ Name: "gcs_testing_storage_location", StorageBaseUrl: gcsBaseUrl, - Encryption: sdk.ExternalVolumeGCSEncryption{ + Encryption: &sdk.ExternalVolumeGCSEncryption{ Type: sdk.GCSEncryptionTypeSseKms, KmsKeyId: &gcsKmsKeyId, }, @@ -239,7 +260,9 @@ func TestInt_ExternalVolumes(t *testing.T) { createExternalVolume := func(t *testing.T, storageLocations []sdk.ExternalVolumeStorageLocation, allowWrites bool, comment string) sdk.AccountObjectIdentifier { t.Helper() - id := testClientHelper().Ids.RandomAccountObjectIdentifier() + //TODO + //id := testClientHelper().Ids.RandomAccountObjectIdentifier() + id := sdk.NewAccountObjectIdentifier("test_external_volume") req := sdk.NewCreateExternalVolumeRequest(id, storageLocations). WithIfNotExists(true). WithAllowWrites(allowWrites). @@ -267,6 +290,17 @@ func TestInt_ExternalVolumes(t *testing.T) { assertExternalVolumeShowResult(t, externalVolume, id, allowWrites, comment) }) + t.Run("Create - S3 Storage Location None Encryption", func(t *testing.T) { + allowWrites := true + comment := "some comment" + id := createExternalVolume(t, s3StorageLocationsNoneEncryption, allowWrites, comment) + + externalVolume, err := client.ExternalVolumes.ShowByID(ctx, id) + require.NoError(t, err) + + assertExternalVolumeShowResult(t, externalVolume, id, allowWrites, comment) + }) + t.Run("Create - S3 Storage Location No Encryption", func(t *testing.T) { allowWrites := true comment := "some comment" @@ -289,6 +323,17 @@ func TestInt_ExternalVolumes(t *testing.T) { assertExternalVolumeShowResult(t, externalVolume, id, allowWrites, comment) }) + t.Run("Create - GCS Storage Location None Encryption", func(t *testing.T) { + allowWrites := true + comment := "some comment" + id := createExternalVolume(t, gcsStorageLocationsNoneEncryption, allowWrites, comment) + + externalVolume, err := client.ExternalVolumes.ShowByID(ctx, id) + require.NoError(t, err) + + assertExternalVolumeShowResult(t, externalVolume, id, allowWrites, comment) + }) + t.Run("Create - GCS Storage Location No Encryption", func(t *testing.T) { allowWrites := true comment := "some comment" @@ -314,7 +359,7 @@ func TestInt_ExternalVolumes(t *testing.T) { t.Run("Create - Multiple Storage Locations", func(t *testing.T) { allowWrites := true comment := "some comment" - id := createExternalVolume(t, append(append(s3StorageLocations, gcsStorageLocationsNoEncryption...), azureStorageLocations...), allowWrites, comment) + id := createExternalVolume(t, append(append(s3StorageLocations, gcsStorageLocationsNoneEncryption...), azureStorageLocations...), allowWrites, comment) externalVolume, err := client.ExternalVolumes.ShowByID(ctx, id) require.NoError(t, err) @@ -325,9 +370,9 @@ func TestInt_ExternalVolumes(t *testing.T) { t.Run("Alter - remove storage location", func(t *testing.T) { allowWrites := true comment := "some comment" - id := createExternalVolume(t, append(s3StorageLocationsNoEncryption, gcsStorageLocationsNoEncryption...), allowWrites, comment) + id := createExternalVolume(t, append(s3StorageLocationsNoneEncryption, gcsStorageLocationsNoneEncryption...), allowWrites, comment) - req := sdk.NewAlterExternalVolumeRequest(id).WithRemoveStorageLocation(gcsStorageLocationsNoEncryption[0].GCSStorageLocationParams.Name) + req := sdk.NewAlterExternalVolumeRequest(id).WithRemoveStorageLocation(gcsStorageLocationsNoneEncryption[0].GCSStorageLocationParams.Name) err := client.ExternalVolumes.Alter(ctx, req) require.NoError(t, err) @@ -347,12 +392,12 @@ func TestInt_ExternalVolumes(t *testing.T) { Name: "STORAGE_LOCATION_1", Value: fmt.Sprintf( `{"NAME":"%s","STORAGE_PROVIDER":"%s","STORAGE_BASE_URL":"%s","STORAGE_AWS_ROLE_ARN":"%s","STORAGE_AWS_EXTERNAL_ID":"%s","ENCRYPTION_TYPE":"%s"}`, - s3StorageLocationsNoEncryption[0].S3StorageLocationParams.Name, - s3StorageLocationsNoEncryption[0].S3StorageLocationParams.StorageProvider, - s3StorageLocationsNoEncryption[0].S3StorageLocationParams.StorageBaseUrl, - s3StorageLocationsNoEncryption[0].S3StorageLocationParams.StorageAwsRoleArn, - *s3StorageLocationsNoEncryption[0].S3StorageLocationParams.StorageAwsExternalId, - s3StorageLocationsNoEncryption[0].S3StorageLocationParams.Encryption.Type, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.Name, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageProvider, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageBaseUrl, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageAwsRoleArn, + *s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageAwsExternalId, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.Encryption.Type, ), }, ) @@ -361,7 +406,7 @@ func TestInt_ExternalVolumes(t *testing.T) { t.Run("Alter - set comment", func(t *testing.T) { allowWrites := true comment := "" - id := createExternalVolume(t, s3StorageLocationsNoEncryption, allowWrites, "some comment") + id := createExternalVolume(t, s3StorageLocationsNoneEncryption, allowWrites, "some comment") req := sdk.NewAlterExternalVolumeRequest(id).WithSet( *sdk.NewAlterExternalVolumeSetRequest().WithComment(comment), @@ -384,12 +429,12 @@ func TestInt_ExternalVolumes(t *testing.T) { Name: "STORAGE_LOCATION_1", Value: fmt.Sprintf( `{"NAME":"%s","STORAGE_PROVIDER":"%s","STORAGE_BASE_URL":"%s","STORAGE_AWS_ROLE_ARN":"%s","STORAGE_AWS_EXTERNAL_ID":"%s","ENCRYPTION_TYPE":"%s"}`, - s3StorageLocationsNoEncryption[0].S3StorageLocationParams.Name, - s3StorageLocationsNoEncryption[0].S3StorageLocationParams.StorageProvider, - s3StorageLocationsNoEncryption[0].S3StorageLocationParams.StorageBaseUrl, - s3StorageLocationsNoEncryption[0].S3StorageLocationParams.StorageAwsRoleArn, - *s3StorageLocationsNoEncryption[0].S3StorageLocationParams.StorageAwsExternalId, - s3StorageLocationsNoEncryption[0].S3StorageLocationParams.Encryption.Type, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.Name, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageProvider, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageBaseUrl, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageAwsRoleArn, + *s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageAwsExternalId, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.Encryption.Type, ), }, ) @@ -437,7 +482,7 @@ func TestInt_ExternalVolumes(t *testing.T) { t.Run("Alter - add s3 storage location to external volume", func(t *testing.T) { allowWrites := true comment := "some comment" - id := createExternalVolume(t, gcsStorageLocationsNoEncryption, allowWrites, comment) + id := createExternalVolume(t, gcsStorageLocationsNoneEncryption, allowWrites, comment) req := sdk.NewAlterExternalVolumeRequest(id).WithAddStorageLocation( *sdk.NewExternalVolumeStorageLocationRequest().WithS3StorageLocationParams( @@ -446,11 +491,12 @@ func TestInt_ExternalVolumes(t *testing.T) { s3StorageLocations[0].S3StorageLocationParams.StorageProvider, s3StorageLocations[0].S3StorageLocationParams.StorageAwsRoleArn, s3StorageLocations[0].S3StorageLocationParams.StorageBaseUrl, - *sdk.NewExternalVolumeS3EncryptionRequest( - s3StorageLocations[0].S3StorageLocationParams.Encryption.Type, - ).WithKmsKeyId(*s3StorageLocations[0].S3StorageLocationParams.Encryption.KmsKeyId), - ).WithStorageAwsExternalId(*s3StorageLocations[0].S3StorageLocationParams.StorageAwsExternalId), - ), + ).WithStorageAwsExternalId(*s3StorageLocations[0].S3StorageLocationParams.StorageAwsExternalId). + WithEncryption( + *sdk.NewExternalVolumeS3EncryptionRequest(s3StorageLocations[0].S3StorageLocationParams.Encryption.Type,). + WithKmsKeyId(*s3StorageLocations[0].S3StorageLocationParams.Encryption.KmsKeyId), + ), + ), ) err := client.ExternalVolumes.Alter(ctx, req) @@ -471,9 +517,9 @@ func TestInt_ExternalVolumes(t *testing.T) { Name: "STORAGE_LOCATION_1", Value: fmt.Sprintf( `{"NAME":"%s","STORAGE_PROVIDER":"GCS","STORAGE_BASE_URL":"%s","ENCRYPTION_TYPE":"%s"}`, - gcsStorageLocationsNoEncryption[0].GCSStorageLocationParams.Name, - gcsStorageLocationsNoEncryption[0].GCSStorageLocationParams.StorageBaseUrl, - gcsStorageLocationsNoEncryption[0].GCSStorageLocationParams.Encryption.Type, + gcsStorageLocationsNoneEncryption[0].GCSStorageLocationParams.Name, + gcsStorageLocationsNoneEncryption[0].GCSStorageLocationParams.StorageBaseUrl, + gcsStorageLocationsNoneEncryption[0].GCSStorageLocationParams.Encryption.Type, ), }, ) @@ -501,7 +547,7 @@ func TestInt_ExternalVolumes(t *testing.T) { comment := "some comment" id := createExternalVolume( t, - append(append(append(append(s3StorageLocations, gcsStorageLocationsNoEncryption...), azureStorageLocations...), s3StorageLocationsNoEncryption...), gcsStorageLocations...), + append(append(append(append(append(append(s3StorageLocations, gcsStorageLocationsNoneEncryption...), azureStorageLocations...), s3StorageLocationsNoneEncryption...), gcsStorageLocations...), s3StorageLocationsNoEncryption...), gcsStorageLocationsNoEncryption...), allowWrites, comment, ) @@ -510,7 +556,7 @@ func TestInt_ExternalVolumes(t *testing.T) { require.NoError(t, err) trimmedProperties := trimProperties(t, props) - assert.Equal(t, 8, len(trimmedProperties)) + assert.Equal(t, 10, len(trimmedProperties)) assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "COMMENT", Value: comment}) assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "ALLOW_WRITES", Value: strconv.FormatBool(allowWrites)}) assert.Contains(t, trimmedProperties, ExternalVolumePropNameValue{Name: "ACTIVE", Value: ""}) @@ -538,9 +584,9 @@ func TestInt_ExternalVolumes(t *testing.T) { Name: "STORAGE_LOCATION_2", Value: fmt.Sprintf( `{"NAME":"%s","STORAGE_PROVIDER":"GCS","STORAGE_BASE_URL":"%s","ENCRYPTION_TYPE":"%s"}`, - gcsStorageLocationsNoEncryption[0].GCSStorageLocationParams.Name, - gcsStorageLocationsNoEncryption[0].GCSStorageLocationParams.StorageBaseUrl, - gcsStorageLocationsNoEncryption[0].GCSStorageLocationParams.Encryption.Type, + gcsStorageLocationsNoneEncryption[0].GCSStorageLocationParams.Name, + gcsStorageLocationsNoneEncryption[0].GCSStorageLocationParams.StorageBaseUrl, + gcsStorageLocationsNoneEncryption[0].GCSStorageLocationParams.Encryption.Type, ), }, ) @@ -564,12 +610,12 @@ func TestInt_ExternalVolumes(t *testing.T) { Name: "STORAGE_LOCATION_4", Value: fmt.Sprintf( `{"NAME":"%s","STORAGE_PROVIDER":"%s","STORAGE_BASE_URL":"%s","STORAGE_AWS_ROLE_ARN":"%s","STORAGE_AWS_EXTERNAL_ID":"%s","ENCRYPTION_TYPE":"%s"}`, - s3StorageLocationsNoEncryption[0].S3StorageLocationParams.Name, - s3StorageLocationsNoEncryption[0].S3StorageLocationParams.StorageProvider, - s3StorageLocationsNoEncryption[0].S3StorageLocationParams.StorageBaseUrl, - s3StorageLocationsNoEncryption[0].S3StorageLocationParams.StorageAwsRoleArn, - *s3StorageLocationsNoEncryption[0].S3StorageLocationParams.StorageAwsExternalId, - s3StorageLocationsNoEncryption[0].S3StorageLocationParams.Encryption.Type, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.Name, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageProvider, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageBaseUrl, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageAwsRoleArn, + *s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.StorageAwsExternalId, + s3StorageLocationsNoneEncryption[0].S3StorageLocationParams.Encryption.Type, ), }, ) @@ -587,6 +633,33 @@ func TestInt_ExternalVolumes(t *testing.T) { ), }, ) + assert.Contains( + t, + trimmedProperties, + ExternalVolumePropNameValue{ + Name: "STORAGE_LOCATION_6", + Value: fmt.Sprintf( + `{"NAME":"%s","STORAGE_PROVIDER":"%s","STORAGE_BASE_URL":"%s","STORAGE_AWS_ROLE_ARN":"%s","STORAGE_AWS_EXTERNAL_ID":"%s","ENCRYPTION_TYPE":"NONE"}`, + s3StorageLocationsNoEncryption[0].S3StorageLocationParams.Name, + s3StorageLocationsNoEncryption[0].S3StorageLocationParams.StorageProvider, + s3StorageLocationsNoEncryption[0].S3StorageLocationParams.StorageBaseUrl, + s3StorageLocationsNoEncryption[0].S3StorageLocationParams.StorageAwsRoleArn, + *s3StorageLocationsNoEncryption[0].S3StorageLocationParams.StorageAwsExternalId, + ), + }, + ) + assert.Contains( + t, + trimmedProperties, + ExternalVolumePropNameValue{ + Name: "STORAGE_LOCATION_7", + Value: fmt.Sprintf( + `{"NAME":"%s","STORAGE_PROVIDER":"GCS","STORAGE_BASE_URL":"%s","ENCRYPTION_TYPE":"NONE"}`, + gcsStorageLocationsNoEncryption[0].GCSStorageLocationParams.Name, + gcsStorageLocationsNoEncryption[0].GCSStorageLocationParams.StorageBaseUrl, + ), + }, + ) }) t.Run("Show with like", func(t *testing.T) { From a757e2553e1732f6f5b036a74a1e9840e336a5b9 Mon Sep 17 00:00:00 2001 From: James Doldissen Date: Tue, 3 Sep 2024 19:24:16 +0200 Subject: [PATCH 05/10] fix: Run linter --- pkg/sdk/external_volumes_def.go | 5 +++-- pkg/sdk/external_volumes_impl_gen.go | 8 -------- .../external_volumes_gen_integration_test.go | 14 +++++++------- 3 files changed, 10 insertions(+), 17 deletions(-) diff --git a/pkg/sdk/external_volumes_def.go b/pkg/sdk/external_volumes_def.go index 56abe176fa..5b554ae9d2 100644 --- a/pkg/sdk/external_volumes_def.go +++ b/pkg/sdk/external_volumes_def.go @@ -52,8 +52,9 @@ var externalAzureStorageLocationDef = g.NewQueryStruct("AzureStorageLocationPara TextAssignment("AZURE_TENANT_ID", g.ParameterOptions().SingleQuotes().Required()). TextAssignment("STORAGE_BASE_URL", g.ParameterOptions().SingleQuotes().Required()) -var storageLocationDef = g.NewQueryStruct("ExternalVolumeStorageLocation"). // Can't name StorageLocation due to naming clash with type in storage integration - OptionalQueryStructField( +// Can't name StorageLocation due to naming clash with type in storage integration +var storageLocationDef = g.NewQueryStruct("ExternalVolumeStorageLocation"). + OptionalQueryStructField( "S3StorageLocationParams", externalS3StorageLocationDef, g.ListOptions().Parentheses().NoComma(), diff --git a/pkg/sdk/external_volumes_impl_gen.go b/pkg/sdk/external_volumes_impl_gen.go index 356f2122bf..7b01131424 100644 --- a/pkg/sdk/external_volumes_impl_gen.go +++ b/pkg/sdk/external_volumes_impl_gen.go @@ -76,12 +76,10 @@ func (r *AlterExternalVolumeRequest) toOpts() *AlterExternalVolumeOptions { } if r.Set != nil { - opts.Set = &AlterExternalVolumeSet{ AllowWrites: r.Set.AllowWrites, Comment: r.Set.Comment, } - } if r.AddStorageLocation != nil { @@ -99,12 +97,10 @@ func (r *AlterExternalVolumeRequest) toOpts() *AlterExternalVolumeOptions { } if r.AddStorageLocation.S3StorageLocationParams.Encryption != nil { - opts.AddStorageLocation.S3StorageLocationParams.Encryption = &ExternalVolumeS3Encryption{ Type: r.AddStorageLocation.S3StorageLocationParams.Encryption.Type, KmsKeyId: r.AddStorageLocation.S3StorageLocationParams.Encryption.KmsKeyId, } - } } @@ -117,24 +113,20 @@ func (r *AlterExternalVolumeRequest) toOpts() *AlterExternalVolumeOptions { } if r.AddStorageLocation.GCSStorageLocationParams.Encryption != nil { - opts.AddStorageLocation.GCSStorageLocationParams.Encryption = &ExternalVolumeGCSEncryption{ Type: r.AddStorageLocation.GCSStorageLocationParams.Encryption.Type, KmsKeyId: r.AddStorageLocation.GCSStorageLocationParams.Encryption.KmsKeyId, } - } } if r.AddStorageLocation.AzureStorageLocationParams != nil { - opts.AddStorageLocation.AzureStorageLocationParams = &AzureStorageLocationParams{ Name: r.AddStorageLocation.AzureStorageLocationParams.Name, AzureTenantId: r.AddStorageLocation.AzureStorageLocationParams.AzureTenantId, StorageBaseUrl: r.AddStorageLocation.AzureStorageLocationParams.StorageBaseUrl, } - } } diff --git a/pkg/sdk/testint/external_volumes_gen_integration_test.go b/pkg/sdk/testint/external_volumes_gen_integration_test.go index ff945f13e6..cdda2ccbcf 100644 --- a/pkg/sdk/testint/external_volumes_gen_integration_test.go +++ b/pkg/sdk/testint/external_volumes_gen_integration_test.go @@ -260,8 +260,8 @@ func TestInt_ExternalVolumes(t *testing.T) { createExternalVolume := func(t *testing.T, storageLocations []sdk.ExternalVolumeStorageLocation, allowWrites bool, comment string) sdk.AccountObjectIdentifier { t.Helper() - //TODO - //id := testClientHelper().Ids.RandomAccountObjectIdentifier() + // TODO + // id := testClientHelper().Ids.RandomAccountObjectIdentifier() id := sdk.NewAccountObjectIdentifier("test_external_volume") req := sdk.NewCreateExternalVolumeRequest(id, storageLocations). WithIfNotExists(true). @@ -492,11 +492,11 @@ func TestInt_ExternalVolumes(t *testing.T) { s3StorageLocations[0].S3StorageLocationParams.StorageAwsRoleArn, s3StorageLocations[0].S3StorageLocationParams.StorageBaseUrl, ).WithStorageAwsExternalId(*s3StorageLocations[0].S3StorageLocationParams.StorageAwsExternalId). - WithEncryption( - *sdk.NewExternalVolumeS3EncryptionRequest(s3StorageLocations[0].S3StorageLocationParams.Encryption.Type,). - WithKmsKeyId(*s3StorageLocations[0].S3StorageLocationParams.Encryption.KmsKeyId), - ), - ), + WithEncryption( + *sdk.NewExternalVolumeS3EncryptionRequest(s3StorageLocations[0].S3StorageLocationParams.Encryption.Type). + WithKmsKeyId(*s3StorageLocations[0].S3StorageLocationParams.Encryption.KmsKeyId), + ), + ), ) err := client.ExternalVolumes.Alter(ctx, req) From 35bd115bb428b7106e9f13f4e2d5320d55ab7f44 Mon Sep 17 00:00:00 2001 From: James Doldissen Date: Tue, 3 Sep 2024 19:30:30 +0200 Subject: [PATCH 06/10] Add test - exactly one field from [opts.RemoveStorageLocation opts.Set opts.AddStorageLocation] should be present - zero set --- pkg/sdk/external_volumes_gen_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/sdk/external_volumes_gen_test.go b/pkg/sdk/external_volumes_gen_test.go index e4b20dbc6a..4278e47db6 100644 --- a/pkg/sdk/external_volumes_gen_test.go +++ b/pkg/sdk/external_volumes_gen_test.go @@ -220,6 +220,12 @@ func TestExternalVolumes_Alter(t *testing.T) { assertOptsInvalidJoinedErrors(t, opts, ErrNilOptions) }) + t.Run("validation: exactly one field from [opts.RemoveStorageLocation opts.Set opts.AddStorageLocation] should be present - zero set", func(t *testing.T) { + opts := defaultOpts() + opts.IfExists = Bool(true) + assertOptsInvalidJoinedErrors(t, opts, errExactlyOneOf("AlterExternalVolumeOptions", "RemoveStorageLocation", "Set", "AddStorageLocation")) + }) + t.Run("validation: exactly one field from [opts.RemoveStorageLocation opts.Set opts.AddStorageLocation] should be present - two set", func(t *testing.T) { removeAndSetOpts := defaultOpts() removeAndAddOpts := defaultOpts() From 07ab8bce6426d5025eb5ea33b16635ae5348545e Mon Sep 17 00:00:00 2001 From: James Doldissen Date: Tue, 3 Sep 2024 19:37:12 +0200 Subject: [PATCH 07/10] fix: Add WithIfExists to drop cleanup function in integration tests --- pkg/sdk/testint/external_volumes_gen_integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/sdk/testint/external_volumes_gen_integration_test.go b/pkg/sdk/testint/external_volumes_gen_integration_test.go index cdda2ccbcf..0db489015f 100644 --- a/pkg/sdk/testint/external_volumes_gen_integration_test.go +++ b/pkg/sdk/testint/external_volumes_gen_integration_test.go @@ -272,7 +272,7 @@ func TestInt_ExternalVolumes(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { - err := client.ExternalVolumes.Drop(ctx, sdk.NewDropExternalVolumeRequest(id)) + err := client.ExternalVolumes.Drop(ctx, sdk.NewDropExternalVolumeRequest(id).WithIfExists(true)) require.NoError(t, err) }) From 623abea08adebfb782a8b7e5c88f45b98d762fec Mon Sep 17 00:00:00 2001 From: James Doldissen Date: Tue, 3 Sep 2024 19:46:17 +0200 Subject: [PATCH 08/10] fix: Use helper methods for for describe operation and show operation types --- pkg/sdk/external_volumes_def.go | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/pkg/sdk/external_volumes_def.go b/pkg/sdk/external_volumes_def.go index 5b554ae9d2..fb4f610be9 100644 --- a/pkg/sdk/external_volumes_def.go +++ b/pkg/sdk/external_volumes_def.go @@ -127,17 +127,17 @@ var ExternalVolumesDef = g.NewInterface( g.DescriptionMappingKindSlice, "https://docs.snowflake.com/en/sql-reference/sql/desc-external-volume", g.DbStruct("externalVolumeDescRow"). - Field("parent_property", "string"). - Field("property", "string"). - Field("property_type", "string"). - Field("property_value", "string"). - Field("property_default", "string"), + Text("parent_property"). + Text("property"). + Text("property_type"). + Text("property_value"). + Text("property_default"), g.PlainStruct("ExternalVolumeProperty"). - Field("Parent", "string"). - Field("Name", "string"). - Field("Type", "string"). - Field("Value", "string"). - Field("Default", "string"), + Text("Parent"). + Text("Name"). + Text("Type"). + Text("Value"). + Text("Default"), g.NewQueryStruct("DescExternalVolume"). Describe(). SQL("EXTERNAL VOLUME"). @@ -147,13 +147,13 @@ var ExternalVolumesDef = g.NewInterface( ShowOperation( "https://docs.snowflake.com/en/sql-reference/sql/show-external-volumes", g.DbStruct("externalVolumeShowRow"). - Field("name", "string"). - Field("allow_writes", "string"). - Field("comment", "string"), + Text("name"). + Text("allow_writes"). + Text("comment"), g.PlainStruct("ExternalVolume"). - Field("Name", "string"). - Field("AllowWrites", "string"). - Field("Comment", "string"), + Text("Name"). + Text("AllowWrites"). + Text("Comment"), g.NewQueryStruct("ShowExternalVolumes"). Show(). SQL("EXTERNAL VOLUMES"). From 5a57e8d0146462d49657ff87bc3c09eccf792c71 Mon Sep 17 00:00:00 2001 From: James Doldissen Date: Tue, 3 Sep 2024 19:52:37 +0200 Subject: [PATCH 09/10] fix: Remove TODO --- pkg/sdk/testint/external_volumes_gen_integration_test.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/sdk/testint/external_volumes_gen_integration_test.go b/pkg/sdk/testint/external_volumes_gen_integration_test.go index 0db489015f..1340cc47ca 100644 --- a/pkg/sdk/testint/external_volumes_gen_integration_test.go +++ b/pkg/sdk/testint/external_volumes_gen_integration_test.go @@ -260,9 +260,7 @@ func TestInt_ExternalVolumes(t *testing.T) { createExternalVolume := func(t *testing.T, storageLocations []sdk.ExternalVolumeStorageLocation, allowWrites bool, comment string) sdk.AccountObjectIdentifier { t.Helper() - // TODO - // id := testClientHelper().Ids.RandomAccountObjectIdentifier() - id := sdk.NewAccountObjectIdentifier("test_external_volume") + id := testClientHelper().Ids.RandomAccountObjectIdentifier() req := sdk.NewCreateExternalVolumeRequest(id, storageLocations). WithIfNotExists(true). WithAllowWrites(allowWrites). From 20808a006e04687aef5c5744d4b07a2e6201169f Mon Sep 17 00:00:00 2001 From: James Doldissen Date: Wed, 4 Sep 2024 14:09:09 +0200 Subject: [PATCH 10/10] fix: Add WithLike to ShowById implementation --- pkg/sdk/external_volumes_impl_gen.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/sdk/external_volumes_impl_gen.go b/pkg/sdk/external_volumes_impl_gen.go index 7b01131424..80b70d934a 100644 --- a/pkg/sdk/external_volumes_impl_gen.go +++ b/pkg/sdk/external_volumes_impl_gen.go @@ -49,7 +49,9 @@ func (v *externalVolumes) Show(ctx context.Context, request *ShowExternalVolumeR } func (v *externalVolumes) ShowByID(ctx context.Context, id AccountObjectIdentifier) (*ExternalVolume, error) { - externalVolumes, err := v.Show(ctx, NewShowExternalVolumeRequest()) + externalVolumes, err := v.Show(ctx, NewShowExternalVolumeRequest().WithLike(Like{ + Pattern: String(id.Name()), + })) if err != nil { return nil, err }