diff --git a/pkg/resources/external_table.go b/pkg/resources/external_table.go index 7a4aad4e8af..cbcf5989760 100644 --- a/pkg/resources/external_table.go +++ b/pkg/resources/external_table.go @@ -30,6 +30,14 @@ var externalTableSchema = map[string]*schema.Schema{ ForceNew: true, Description: "The database in which to create the external table.", }, + // TODO: Could be a string that we would validate as always "delta" (could be easy to add another type if snowflake introduces one) + "table_format_delta": { + Type: schema.TypeBool, + Required: true, + ForceNew: true, + Description: `Identifies the external table as referencing a Delta Lake on the cloud storage location. A Delta Lake on Amazon S3, Google Cloud Storage, or Microsoft Azure cloud storage is supported.`, + RequiredWith: []string{"user_specified_partitions"}, + }, "column": { Type: schema.TypeList, Required: true, @@ -84,11 +92,18 @@ var externalTableSchema = map[string]*schema.Schema{ ForceNew: true, Description: "Specifies the aws sns topic for the external table.", }, - "partition_by": { - Type: schema.TypeList, + "user_specified_partitions": { + Type: schema.TypeBool, Optional: true, - Elem: &schema.Schema{Type: schema.TypeString}, ForceNew: true, + Description: "Enables to manage partitions manually and perform updates instead of recreating table on partition_by change.", + }, + "partition_by": { + Type: schema.TypeList, + Optional: true, + Elem: &schema.Schema{Type: schema.TypeString}, + //ForceNew: true, + // TODO: Update on user_specified_partitions = true and force new on false Description: "Specifies any partition columns to evaluate for the external table.", }, "refresh_on_create": { @@ -152,7 +167,6 @@ func CreateExternalTable(d *schema.ResourceData, meta any) error { id := sdk.NewSchemaObjectIdentifier(database, schema, name) location := d.Get("location").(string) fileFormat := d.Get("file_format").(string) - req := sdk.NewCreateExternalTableRequest(id, location).WithRawFileFormat(&fileFormat) tableColumns := d.Get("column").([]any) columnRequests := make([]*sdk.ExternalTableColumnRequest, len(tableColumns)) @@ -171,41 +185,92 @@ func CreateExternalTable(d *schema.ResourceData, meta any) error { as := columnDef["as"] columnRequests[i] = sdk.NewExternalTableColumnRequest(name, dataType, as) } - req.WithColumns(columnRequests) - - req.WithAutoRefresh(sdk.Bool(d.Get("auto_refresh").(bool))) - req.WithRefreshOnCreate(sdk.Bool(d.Get("refresh_on_create").(bool))) - req.WithCopyGrants(sdk.Bool(d.Get("copy_grants").(bool))) + autoRefresh := sdk.Bool(d.Get("auto_refresh").(bool)) + refreshOnCreate := sdk.Bool(d.Get("refresh_on_create").(bool)) + copyGrants := sdk.Bool(d.Get("copy_grants").(bool)) + var partitionBy []string if v, ok := d.GetOk("partition_by"); ok { - partitionBy := expandStringList(v.([]any)) - req.WithPartitionBy(partitionBy) + partitionBy = expandStringList(v.([]any)) } + var pattern *string if v, ok := d.GetOk("pattern"); ok { - req.WithPattern(sdk.String(v.(string))) + pattern = sdk.String(v.(string)) } + var awsSnsTopic *string if v, ok := d.GetOk("aws_sns_topic"); ok { - req.WithAwsSnsTopic(sdk.String(v.(string))) + awsSnsTopic = sdk.String(v.(string)) } + var comment *string if v, ok := d.GetOk("comment"); ok { - req.WithComment(sdk.String(v.(string))) + comment = sdk.String(v.(string)) } + var tagAssociationRequests []*sdk.TagAssociationRequest if _, ok := d.GetOk("tag"); ok { tagAssociations := getPropertyTags(d, "tag") - tagAssociationRequests := make([]*sdk.TagAssociationRequest, len(tagAssociations)) + tagAssociationRequests = make([]*sdk.TagAssociationRequest, len(tagAssociations)) for i, t := range tagAssociations { tagAssociationRequests[i] = sdk.NewTagAssociationRequest(t.Name, t.Value) } - req.WithTag(tagAssociationRequests) } - if err := client.ExternalTables.Create(ctx, req); err != nil { - return err + switch { + case d.Get("table_format_delta").(bool): + err := client.ExternalTables.CreateDeltaLake( + ctx, + sdk.NewCreateDeltaLakeExternalTableRequest(id, location). + WithRawFileFormat(&fileFormat). + WithColumns(columnRequests). + WithPartitionBy(partitionBy). + WithRefreshOnCreate(refreshOnCreate). + WithAutoRefresh(autoRefresh). + WithCopyGrants(copyGrants). + WithComment(comment). + WithTag(tagAssociationRequests), + ) + if err != nil { + return err + } + case d.Get("user_specified_partitions").(bool): + err := client.ExternalTables.CreateWithManualPartitioning( + ctx, + sdk.NewCreateWithManualPartitioningExternalTableRequest(id, location). + WithRawFileFormat(&fileFormat). + WithColumns(columnRequests). + WithPartitionBy(partitionBy). + WithRawFileFormat(sdk.String(fileFormat)). + WithCopyGrants(copyGrants). + WithComment(comment). + WithTag(tagAssociationRequests), + ) + if err != nil { + return err + } + default: + err := client.ExternalTables.Create( + ctx, + sdk.NewCreateExternalTableRequest(id, location). + WithRawFileFormat(&fileFormat). + WithColumns(columnRequests). + WithPartitionBy(partitionBy). + WithRefreshOnCreate(refreshOnCreate). + WithAutoRefresh(autoRefresh). + WithPattern(pattern). + WithRawFileFormat(sdk.String(fileFormat)). + WithAwsSnsTopic(awsSnsTopic). + WithCopyGrants(copyGrants). + WithComment(comment). + WithTag(tagAssociationRequests), + ) + if err != nil { + return err + } } + d.SetId(helpers.EncodeSnowflakeID(id)) return ReadExternalTable(d, meta) diff --git a/pkg/resources/external_table_acceptance_test.go b/pkg/resources/external_table_acceptance_test.go index 1cb407b1265..b990adc75d9 100644 --- a/pkg/resources/external_table_acceptance_test.go +++ b/pkg/resources/external_table_acceptance_test.go @@ -73,6 +73,10 @@ func TestAcc_ExternalTable_basic(t *testing.T) { }) } +// TODO TEST: with partitionBy set (check with select get_ddl) - https://github.com/Snowflake-Labs/terraform-provider-snowflake/issues/2293 +// TODO: support table_format = delta - https://github.com/Snowflake-Labs/terraform-provider-snowflake/issues/1564 +// TODO: invalid column types - https://github.com/Snowflake-Labs/terraform-provider-snowflake/issues/2310 + func testAccCheckExternalTableDestroy(s *terraform.State) error { db := acc.TestAccProvider.Meta().(*sql.DB) client := sdk.NewClientFromDB(db) diff --git a/pkg/sdk/external_tables.go b/pkg/sdk/external_tables.go index fdf5bc53784..b3171eff4c5 100644 --- a/pkg/sdk/external_tables.go +++ b/pkg/sdk/external_tables.go @@ -211,7 +211,7 @@ type CreateWithManualPartitioningExternalTableOptions struct { CloudProviderParams *CloudProviderParams PartitionBy []string `ddl:"keyword,parentheses" sql:"PARTITION BY"` Location string `ddl:"parameter" sql:"LOCATION"` - UserSpecifiedPartitionType *bool `ddl:"keyword" sql:"PARTITION_TYPE = USER_SPECIFIED"` + userSpecifiedPartitionType bool `ddl:"static" sql:"PARTITION_TYPE = USER_SPECIFIED"` FileFormat []ExternalTableFileFormat `ddl:"parameter,parentheses" sql:"FILE_FORMAT"` // RawFileFormat was introduced, because of the decision taken during https://github.com/Snowflake-Labs/terraform-provider-snowflake/pull/2228 // that for now the snowflake_external_table resource should continue on using raw file format, which wasn't previously supported by the new SDK. @@ -236,7 +236,7 @@ type CreateDeltaLakeExternalTableOptions struct { Location string `ddl:"parameter" sql:"LOCATION"` RefreshOnCreate *bool `ddl:"parameter" sql:"REFRESH_ON_CREATE"` AutoRefresh *bool `ddl:"parameter" sql:"AUTO_REFRESH"` - UserSpecifiedPartitionType *bool `ddl:"keyword" sql:"PARTITION_TYPE = USER_SPECIFIED"` + userSpecifiedPartitionType bool `ddl:"static" sql:"PARTITION_TYPE = USER_SPECIFIED"` FileFormat []ExternalTableFileFormat `ddl:"parameter,parentheses" sql:"FILE_FORMAT"` // RawFileFormat was introduced, because of the decision taken during https://github.com/Snowflake-Labs/terraform-provider-snowflake/pull/2228 // that for now the snowflake_external_table resource should continue on using raw file format, which wasn't previously supported by the new SDK. diff --git a/pkg/sdk/external_tables_dto.go b/pkg/sdk/external_tables_dto.go index d27b0850d3b..0d5bf56d78a 100644 --- a/pkg/sdk/external_tables_dto.go +++ b/pkg/sdk/external_tables_dto.go @@ -268,6 +268,7 @@ func (s *CreateExternalTableRequest) toOpts() *CreateExternalTableOptions { name: s.name, Columns: columns, CloudProviderParams: cloudProviderParams, + PartitionBy: s.partitionBy, Location: s.location, RefreshOnCreate: s.refreshOnCreate, AutoRefresh: s.autoRefresh, @@ -283,20 +284,19 @@ func (s *CreateExternalTableRequest) toOpts() *CreateExternalTableOptions { } type CreateWithManualPartitioningExternalTableRequest struct { - orReplace *bool - ifNotExists *bool - name SchemaObjectIdentifier // required - columns []*ExternalTableColumnRequest - cloudProviderParams *CloudProviderParamsRequest - partitionBy []string - location string // required - userSpecifiedPartitionType *bool - rawFileFormat *string - fileFormat *ExternalTableFileFormatRequest - copyGrants *bool - comment *string - rowAccessPolicy *RowAccessPolicyRequest - tag []*TagAssociationRequest + orReplace *bool + ifNotExists *bool + name SchemaObjectIdentifier // required + columns []*ExternalTableColumnRequest + cloudProviderParams *CloudProviderParamsRequest + partitionBy []string + location string // required + rawFileFormat *string + fileFormat *ExternalTableFileFormatRequest + copyGrants *bool + comment *string + rowAccessPolicy *RowAccessPolicyRequest + tag []*TagAssociationRequest } func (v *CreateWithManualPartitioningExternalTableRequest) toOpts() *CreateWithManualPartitioningExternalTableOptions { @@ -337,41 +337,39 @@ func (v *CreateWithManualPartitioningExternalTableRequest) toOpts() *CreateWithM } return &CreateWithManualPartitioningExternalTableOptions{ - OrReplace: v.orReplace, - IfNotExists: v.ifNotExists, - name: v.name, - Columns: columns, - CloudProviderParams: cloudProviderParams, - PartitionBy: v.partitionBy, - Location: v.location, - UserSpecifiedPartitionType: v.userSpecifiedPartitionType, - RawFileFormat: rawFileFormat, - FileFormat: fileFormat, - CopyGrants: v.copyGrants, - Comment: v.comment, - RowAccessPolicy: rowAccessPolicy, - Tag: tag, + OrReplace: v.orReplace, + IfNotExists: v.ifNotExists, + name: v.name, + Columns: columns, + CloudProviderParams: cloudProviderParams, + PartitionBy: v.partitionBy, + Location: v.location, + RawFileFormat: rawFileFormat, + FileFormat: fileFormat, + CopyGrants: v.copyGrants, + Comment: v.comment, + RowAccessPolicy: rowAccessPolicy, + Tag: tag, } } type CreateDeltaLakeExternalTableRequest struct { - orReplace *bool - ifNotExists *bool - name SchemaObjectIdentifier // required - columns []*ExternalTableColumnRequest - cloudProviderParams *CloudProviderParamsRequest - partitionBy []string - location string // required - userSpecifiedPartitionType *bool - refreshOnCreate *bool - autoRefresh *bool - rawFileFormat *string - fileFormat *ExternalTableFileFormatRequest - deltaTableFormat *bool - copyGrants *bool - comment *string - rowAccessPolicy *RowAccessPolicyRequest - tag []*TagAssociationRequest + orReplace *bool + ifNotExists *bool + name SchemaObjectIdentifier // required + columns []*ExternalTableColumnRequest + cloudProviderParams *CloudProviderParamsRequest + partitionBy []string + location string // required + refreshOnCreate *bool + autoRefresh *bool + rawFileFormat *string + fileFormat *ExternalTableFileFormatRequest + deltaTableFormat *bool + copyGrants *bool + comment *string + rowAccessPolicy *RowAccessPolicyRequest + tag []*TagAssociationRequest } func (v *CreateDeltaLakeExternalTableRequest) toOpts() *CreateDeltaLakeExternalTableOptions { @@ -412,23 +410,22 @@ func (v *CreateDeltaLakeExternalTableRequest) toOpts() *CreateDeltaLakeExternalT } return &CreateDeltaLakeExternalTableOptions{ - OrReplace: v.orReplace, - IfNotExists: v.ifNotExists, - name: v.name, - Columns: columns, - CloudProviderParams: cloudProviderParams, - PartitionBy: v.partitionBy, - Location: v.location, - UserSpecifiedPartitionType: v.userSpecifiedPartitionType, - RefreshOnCreate: v.refreshOnCreate, - AutoRefresh: v.autoRefresh, - RawFileFormat: rawFileFormat, - FileFormat: fileFormat, - DeltaTableFormat: v.deltaTableFormat, - CopyGrants: v.copyGrants, - Comment: v.comment, - RowAccessPolicy: rowAccessPolicy, - Tag: tag, + OrReplace: v.orReplace, + IfNotExists: v.ifNotExists, + name: v.name, + Columns: columns, + CloudProviderParams: cloudProviderParams, + PartitionBy: v.partitionBy, + Location: v.location, + RefreshOnCreate: v.refreshOnCreate, + AutoRefresh: v.autoRefresh, + RawFileFormat: rawFileFormat, + FileFormat: fileFormat, + DeltaTableFormat: v.deltaTableFormat, + CopyGrants: v.copyGrants, + Comment: v.comment, + RowAccessPolicy: rowAccessPolicy, + Tag: tag, } } diff --git a/pkg/sdk/external_tables_dto_builders_gen.go b/pkg/sdk/external_tables_dto_builders_gen.go index c4a40af879a..81927008628 100644 --- a/pkg/sdk/external_tables_dto_builders_gen.go +++ b/pkg/sdk/external_tables_dto_builders_gen.go @@ -275,16 +275,6 @@ func (s *NullStringRequest) WithStr(str string) *NullStringRequest { return s } -func NewRowAccessPolicyRequest( - name SchemaObjectIdentifier, - on []string, -) *RowAccessPolicyRequest { - s := RowAccessPolicyRequest{} - s.Name = name - s.On = on - return &s -} - func NewCreateWithManualPartitioningExternalTableRequest( name SchemaObjectIdentifier, location string, @@ -320,11 +310,6 @@ func (s *CreateWithManualPartitioningExternalTableRequest) WithPartitionBy(parti return s } -func (s *CreateWithManualPartitioningExternalTableRequest) WithUserSpecifiedPartitionType(userSpecifiedPartitionType *bool) *CreateWithManualPartitioningExternalTableRequest { - s.userSpecifiedPartitionType = userSpecifiedPartitionType - return s -} - func (s *CreateWithManualPartitioningExternalTableRequest) WithRawFileFormat(rawFileFormat *string) *CreateWithManualPartitioningExternalTableRequest { s.rawFileFormat = rawFileFormat return s @@ -390,11 +375,6 @@ func (s *CreateDeltaLakeExternalTableRequest) WithPartitionBy(partitionBy []stri return s } -func (s *CreateDeltaLakeExternalTableRequest) WithUserSpecifiedPartitionType(userSpecifiedPartitionType *bool) *CreateDeltaLakeExternalTableRequest { - s.userSpecifiedPartitionType = userSpecifiedPartitionType - return s -} - func (s *CreateDeltaLakeExternalTableRequest) WithRefreshOnCreate(refreshOnCreate *bool) *CreateDeltaLakeExternalTableRequest { s.refreshOnCreate = refreshOnCreate return s