wip

pkg/resources/external_table.go

@@ -30,6 +30,14 @@ var externalTableSchema = map[string]*schema.Schema{
 		ForceNew:    true,
 		Description: "The database in which to create the external table.",
 	},
+	// TODO: Could be a string that we would validate as always "delta" (could be easy to add another type if snowflake introduces one)
+	"table_format_delta": {
+		Type:         schema.TypeBool,
+		Required:     true,
+		ForceNew:     true,
+		Description:  `Identifies the external table as referencing a Delta Lake on the cloud storage location. A Delta Lake on Amazon S3, Google Cloud Storage, or Microsoft Azure cloud storage is supported.`,
+		RequiredWith: []string{"user_specified_partitions"},
+	},
 	"column": {
 		Type:        schema.TypeList,
 		Required:    true,
@@ -84,11 +92,18 @@ var externalTableSchema = map[string]*schema.Schema{
 		ForceNew:    true,
 		Description: "Specifies the aws sns topic for the external table.",
 	},
-	"partition_by": {
-		Type:        schema.TypeList,
+	"user_specified_partitions": {
+		Type:        schema.TypeBool,
 		Optional:    true,
-		Elem:        &schema.Schema{Type: schema.TypeString},
 		ForceNew:    true,
+		Description: "Enables to manage partitions manually and perform updates instead of recreating table on partition_by change.",
+	},
+	"partition_by": {
+		Type:     schema.TypeList,
+		Optional: true,
+		Elem:     &schema.Schema{Type: schema.TypeString},
+		//ForceNew:    true,
+		// TODO: Update on user_specified_partitions = true and force new on false
 		Description: "Specifies any partition columns to evaluate for the external table.",
 	},
 	"refresh_on_create": {
@@ -152,7 +167,6 @@ func CreateExternalTable(d *schema.ResourceData, meta any) error {
 	id := sdk.NewSchemaObjectIdentifier(database, schema, name)
 	location := d.Get("location").(string)
 	fileFormat := d.Get("file_format").(string)
-	req := sdk.NewCreateExternalTableRequest(id, location).WithRawFileFormat(&fileFormat)
 
 	tableColumns := d.Get("column").([]any)
 	columnRequests := make([]*sdk.ExternalTableColumnRequest, len(tableColumns))
@@ -171,41 +185,92 @@ func CreateExternalTable(d *schema.ResourceData, meta any) error {
 		as := columnDef["as"]
 		columnRequests[i] = sdk.NewExternalTableColumnRequest(name, dataType, as)
 	}
-	req.WithColumns(columnRequests)
-
-	req.WithAutoRefresh(sdk.Bool(d.Get("auto_refresh").(bool)))
-	req.WithRefreshOnCreate(sdk.Bool(d.Get("refresh_on_create").(bool)))
-	req.WithCopyGrants(sdk.Bool(d.Get("copy_grants").(bool)))
+	autoRefresh := sdk.Bool(d.Get("auto_refresh").(bool))
+	refreshOnCreate := sdk.Bool(d.Get("refresh_on_create").(bool))
+	copyGrants := sdk.Bool(d.Get("copy_grants").(bool))
 
+	var partitionBy []string
 	if v, ok := d.GetOk("partition_by"); ok {
-		partitionBy := expandStringList(v.([]any))
-		req.WithPartitionBy(partitionBy)
+		partitionBy = expandStringList(v.([]any))
 	}
 
+	var pattern *string
 	if v, ok := d.GetOk("pattern"); ok {
-		req.WithPattern(sdk.String(v.(string)))
+		pattern = sdk.String(v.(string))
 	}
 
+	var awsSnsTopic *string
 	if v, ok := d.GetOk("aws_sns_topic"); ok {
-		req.WithAwsSnsTopic(sdk.String(v.(string)))
+		awsSnsTopic = sdk.String(v.(string))
 	}
 
+	var comment *string
 	if v, ok := d.GetOk("comment"); ok {
-		req.WithComment(sdk.String(v.(string)))
+		comment = sdk.String(v.(string))
 	}
 
+	var tagAssociationRequests []*sdk.TagAssociationRequest
 	if _, ok := d.GetOk("tag"); ok {
 		tagAssociations := getPropertyTags(d, "tag")
-		tagAssociationRequests := make([]*sdk.TagAssociationRequest, len(tagAssociations))
+		tagAssociationRequests = make([]*sdk.TagAssociationRequest, len(tagAssociations))
 		for i, t := range tagAssociations {
 			tagAssociationRequests[i] = sdk.NewTagAssociationRequest(t.Name, t.Value)
 		}
-		req.WithTag(tagAssociationRequests)
 	}
 
-	if err := client.ExternalTables.Create(ctx, req); err != nil {
-		return err
+	switch {
+	case d.Get("table_format_delta").(bool):
+		err := client.ExternalTables.CreateDeltaLake(
+			ctx,
+			sdk.NewCreateDeltaLakeExternalTableRequest(id, location).
+				WithRawFileFormat(&fileFormat).
+				WithColumns(columnRequests).
+				WithPartitionBy(partitionBy).
+				WithRefreshOnCreate(refreshOnCreate).
+				WithAutoRefresh(autoRefresh).
+				WithCopyGrants(copyGrants).
+				WithComment(comment).
+				WithTag(tagAssociationRequests),
+		)
+		if err != nil {
+			return err
+		}
+	case d.Get("user_specified_partitions").(bool):
+		err := client.ExternalTables.CreateWithManualPartitioning(
+			ctx,
+			sdk.NewCreateWithManualPartitioningExternalTableRequest(id, location).
+				WithRawFileFormat(&fileFormat).
+				WithColumns(columnRequests).
+				WithPartitionBy(partitionBy).
+				WithRawFileFormat(sdk.String(fileFormat)).
+				WithCopyGrants(copyGrants).
+				WithComment(comment).
+				WithTag(tagAssociationRequests),
+		)
+		if err != nil {
+			return err
+		}
+	default:
+		err := client.ExternalTables.Create(
+			ctx,
+			sdk.NewCreateExternalTableRequest(id, location).
+				WithRawFileFormat(&fileFormat).
+				WithColumns(columnRequests).
+				WithPartitionBy(partitionBy).
+				WithRefreshOnCreate(refreshOnCreate).
+				WithAutoRefresh(autoRefresh).
+				WithPattern(pattern).
+				WithRawFileFormat(sdk.String(fileFormat)).
+				WithAwsSnsTopic(awsSnsTopic).
+				WithCopyGrants(copyGrants).
+				WithComment(comment).
+				WithTag(tagAssociationRequests),
+		)
+		if err != nil {
+			return err
+		}
 	}
+
 	d.SetId(helpers.EncodeSnowflakeID(id))
 
 	return ReadExternalTable(d, meta)

pkg/resources/external_table_acceptance_test.go

@@ -73,6 +73,10 @@ func TestAcc_ExternalTable_basic(t *testing.T) {
 	})
 }
 
+// TODO TEST: with partitionBy set (check with select get_ddl) - https://github.com/Snowflake-Labs/terraform-provider-snowflake/issues/2293
+// TODO: support table_format = delta - https://github.com/Snowflake-Labs/terraform-provider-snowflake/issues/1564
+// TODO: invalid column types - https://github.com/Snowflake-Labs/terraform-provider-snowflake/issues/2310
+
 func testAccCheckExternalTableDestroy(s *terraform.State) error {
 	db := acc.TestAccProvider.Meta().(*sql.DB)
 	client := sdk.NewClientFromDB(db)

pkg/sdk/external_tables.go

@@ -211,7 +211,7 @@ type CreateWithManualPartitioningExternalTableOptions struct {
 	CloudProviderParams        *CloudProviderParams
 	PartitionBy                []string                  `ddl:"keyword,parentheses" sql:"PARTITION BY"`
 	Location                   string                    `ddl:"parameter" sql:"LOCATION"`
-	UserSpecifiedPartitionType *bool                     `ddl:"keyword" sql:"PARTITION_TYPE = USER_SPECIFIED"`
+	userSpecifiedPartitionType bool                      `ddl:"static" sql:"PARTITION_TYPE = USER_SPECIFIED"`
 	FileFormat                 []ExternalTableFileFormat `ddl:"parameter,parentheses" sql:"FILE_FORMAT"`
 	// RawFileFormat was introduced, because of the decision taken during https://github.com/Snowflake-Labs/terraform-provider-snowflake/pull/2228
 	// that for now the snowflake_external_table resource should continue on using raw file format, which wasn't previously supported by the new SDK.
@@ -236,7 +236,7 @@ type CreateDeltaLakeExternalTableOptions struct {
 	Location                   string                    `ddl:"parameter" sql:"LOCATION"`
 	RefreshOnCreate            *bool                     `ddl:"parameter" sql:"REFRESH_ON_CREATE"`
 	AutoRefresh                *bool                     `ddl:"parameter" sql:"AUTO_REFRESH"`
-	UserSpecifiedPartitionType *bool                     `ddl:"keyword" sql:"PARTITION_TYPE = USER_SPECIFIED"`
+	userSpecifiedPartitionType bool                      `ddl:"static" sql:"PARTITION_TYPE = USER_SPECIFIED"`
 	FileFormat                 []ExternalTableFileFormat `ddl:"parameter,parentheses" sql:"FILE_FORMAT"`
 	// RawFileFormat was introduced, because of the decision taken during https://github.com/Snowflake-Labs/terraform-provider-snowflake/pull/2228
 	// that for now the snowflake_external_table resource should continue on using raw file format, which wasn't previously supported by the new SDK.

pkg/sdk/external_tables_dto.go

@@ -268,6 +268,7 @@ func (s *CreateExternalTableRequest) toOpts() *CreateExternalTableOptions {
 		name:                s.name,
 		Columns:             columns,
 		CloudProviderParams: cloudProviderParams,
+		PartitionBy:         s.partitionBy,
 		Location:            s.location,
 		RefreshOnCreate:     s.refreshOnCreate,
 		AutoRefresh:         s.autoRefresh,
@@ -283,20 +284,19 @@ func (s *CreateExternalTableRequest) toOpts() *CreateExternalTableOptions {
 }
 
 type CreateWithManualPartitioningExternalTableRequest struct {
-	orReplace                  *bool
-	ifNotExists                *bool
-	name                       SchemaObjectIdentifier // required
-	columns                    []*ExternalTableColumnRequest
-	cloudProviderParams        *CloudProviderParamsRequest
-	partitionBy                []string
-	location                   string // required
-	userSpecifiedPartitionType *bool
-	rawFileFormat              *string
-	fileFormat                 *ExternalTableFileFormatRequest
-	copyGrants                 *bool
-	comment                    *string
-	rowAccessPolicy            *RowAccessPolicyRequest
-	tag                        []*TagAssociationRequest
+	orReplace           *bool
+	ifNotExists         *bool
+	name                SchemaObjectIdentifier // required
+	columns             []*ExternalTableColumnRequest
+	cloudProviderParams *CloudProviderParamsRequest
+	partitionBy         []string
+	location            string // required
+	rawFileFormat       *string
+	fileFormat          *ExternalTableFileFormatRequest
+	copyGrants          *bool
+	comment             *string
+	rowAccessPolicy     *RowAccessPolicyRequest
+	tag                 []*TagAssociationRequest
 }
 
 func (v *CreateWithManualPartitioningExternalTableRequest) toOpts() *CreateWithManualPartitioningExternalTableOptions {
@@ -337,41 +337,39 @@ func (v *CreateWithManualPartitioningExternalTableRequest) toOpts() *CreateWithM
 	}
 
 	return &CreateWithManualPartitioningExternalTableOptions{
-		OrReplace:                  v.orReplace,
-		IfNotExists:                v.ifNotExists,
-		name:                       v.name,
-		Columns:                    columns,
-		CloudProviderParams:        cloudProviderParams,
-		PartitionBy:                v.partitionBy,
-		Location:                   v.location,
-		UserSpecifiedPartitionType: v.userSpecifiedPartitionType,
-		RawFileFormat:              rawFileFormat,
-		FileFormat:                 fileFormat,
-		CopyGrants:                 v.copyGrants,
-		Comment:                    v.comment,
-		RowAccessPolicy:            rowAccessPolicy,
-		Tag:                        tag,
+		OrReplace:           v.orReplace,
+		IfNotExists:         v.ifNotExists,
+		name:                v.name,
+		Columns:             columns,
+		CloudProviderParams: cloudProviderParams,
+		PartitionBy:         v.partitionBy,
+		Location:            v.location,
+		RawFileFormat:       rawFileFormat,
+		FileFormat:          fileFormat,
+		CopyGrants:          v.copyGrants,
+		Comment:             v.comment,
+		RowAccessPolicy:     rowAccessPolicy,
+		Tag:                 tag,
 	}
 }
 
 type CreateDeltaLakeExternalTableRequest struct {
-	orReplace                  *bool
-	ifNotExists                *bool
-	name                       SchemaObjectIdentifier // required
-	columns                    []*ExternalTableColumnRequest
-	cloudProviderParams        *CloudProviderParamsRequest
-	partitionBy                []string
-	location                   string // required
-	userSpecifiedPartitionType *bool
-	refreshOnCreate            *bool
-	autoRefresh                *bool
-	rawFileFormat              *string
-	fileFormat                 *ExternalTableFileFormatRequest
-	deltaTableFormat           *bool
-	copyGrants                 *bool
-	comment                    *string
-	rowAccessPolicy            *RowAccessPolicyRequest
-	tag                        []*TagAssociationRequest
+	orReplace           *bool
+	ifNotExists         *bool
+	name                SchemaObjectIdentifier // required
+	columns             []*ExternalTableColumnRequest
+	cloudProviderParams *CloudProviderParamsRequest
+	partitionBy         []string
+	location            string // required
+	refreshOnCreate     *bool
+	autoRefresh         *bool
+	rawFileFormat       *string
+	fileFormat          *ExternalTableFileFormatRequest
+	deltaTableFormat    *bool
+	copyGrants          *bool
+	comment             *string
+	rowAccessPolicy     *RowAccessPolicyRequest
+	tag                 []*TagAssociationRequest
 }
 
 func (v *CreateDeltaLakeExternalTableRequest) toOpts() *CreateDeltaLakeExternalTableOptions {
@@ -412,23 +410,22 @@ func (v *CreateDeltaLakeExternalTableRequest) toOpts() *CreateDeltaLakeExternalT
 	}
 
 	return &CreateDeltaLakeExternalTableOptions{
-		OrReplace:                  v.orReplace,
-		IfNotExists:                v.ifNotExists,
-		name:                       v.name,
-		Columns:                    columns,
-		CloudProviderParams:        cloudProviderParams,
-		PartitionBy:                v.partitionBy,
-		Location:                   v.location,
-		UserSpecifiedPartitionType: v.userSpecifiedPartitionType,
-		RefreshOnCreate:            v.refreshOnCreate,
-		AutoRefresh:                v.autoRefresh,
-		RawFileFormat:              rawFileFormat,
-		FileFormat:                 fileFormat,
-		DeltaTableFormat:           v.deltaTableFormat,
-		CopyGrants:                 v.copyGrants,
-		Comment:                    v.comment,
-		RowAccessPolicy:            rowAccessPolicy,
-		Tag:                        tag,
+		OrReplace:           v.orReplace,
+		IfNotExists:         v.ifNotExists,
+		name:                v.name,
+		Columns:             columns,
+		CloudProviderParams: cloudProviderParams,
+		PartitionBy:         v.partitionBy,
+		Location:            v.location,
+		RefreshOnCreate:     v.refreshOnCreate,
+		AutoRefresh:         v.autoRefresh,
+		RawFileFormat:       rawFileFormat,
+		FileFormat:          fileFormat,
+		DeltaTableFormat:    v.deltaTableFormat,
+		CopyGrants:          v.copyGrants,
+		Comment:             v.comment,
+		RowAccessPolicy:     rowAccessPolicy,
+		Tag:                 tag,
 	}
 }