feat: add refresh_mode and initialize to dynamic tables (#2437)

This PR contains two commits that add `refresh_mode` and `initialize`
parameters to the `snowflake_dynamic_table` resource.

`REFRESH_MODE` and `INITIALIZE` are parameters that can only be set at
creation time. (I believe these were both added within the last couple
months.) They are listed under [required
parameters](https://docs.snowflake.com/en/sql-reference/sql/create-dynamic-table#required-parameters)
in the documentation, but both have defaults. There are use cases where,
for performance or other reasons, users need to be able to override the
defaults.

This PR adds support for these two options by changing `refresh_mode`
from a read-only parameter to a settable optional parameter, and adding
`initialize` as a new parameter. Please see the individual commit
messages for additional caveats and implementation notes.

I am opening this PR because we have ongoing development that is blocked
without the ability to set these parameters in Terraform. I realize this
repo is in the middle of a major framework update, so please let me know
if there's anything I can do to help make these features available
without being disruptive to the framework work.

## Test Plan
* [x] acceptance tests
* [x] testing with our existing stack. We have a stack that includes
~100 dynamic tables, and I have been running `terraform apply` with this
code change in place. This should provide decent coverage of use cases.

docs/resources/dynamic_table.md

@@ -3,7 +3,7 @@
 page_title: "snowflake_dynamic_table Resource - terraform-provider-snowflake"
 subcategory: ""
 description: |-
-  
+
 ---
 
 # snowflake_dynamic_table (Resource)
@@ -43,6 +43,8 @@ resource "snowflake_dynamic_table" "dt" {
 
 - `comment` (String) Specifies a comment for the dynamic table.
 - `or_replace` (Boolean) Specifies whether to replace the dynamic table if it already exists.
+- `refresh_mode` (String) INCREMENTAL if the dynamic table will use incremental refreshes, or FULL if it will recompute the whole table on every refresh. Specify AUTO to let Snowflake decide. The default is AUTO.
+- `initialize` (String) Specifies the behavior of the initial refresh of the dynamic table. This property cannot be altered after you create the dynamic table. Specify ON_CREATE to initialize the dynamic table immeidately, or ON_SCHEDULE to have it initialize at the next tick after creation. The default os ON_CREATE.
 
 ### Read-Only
 
@@ -55,7 +57,6 @@ resource "snowflake_dynamic_table" "dt" {
 - `is_replica` (Boolean) TRUE if the dynamic table is a replica. else FALSE.
 - `last_suspended_on` (String) Timestamp of last suspension.
 - `owner` (String) Role that owns the dynamic table.
-- `refresh_mode` (String) INCREMENTAL if the dynamic table will use incremental refreshes, or FULL if it will recompute the whole table on every refresh.
 - `refresh_mode_reason` (String) Explanation for why FULL refresh mode was chosen. NULL if refresh mode is not FULL.
 - `rows` (Number) Number of rows in the table.
 - `scheduling_state` (String) Displays RUNNING for dynamic tables that are actively scheduling refreshes and SUSPENDED for suspended dynamic tables.

pkg/resources/dynamic_table.go

@@ -4,14 +4,19 @@ import (
 	"context"
 	"database/sql"
 	"log"
+	"regexp"
 	"strings"
+	"time"
 
 	"github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/helpers"
 	"github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/sdk"
 	"github.com/Snowflake-Labs/terraform-provider-snowflake/pkg/snowflake"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation"
 )
 
+var refreshModePattern = regexp.MustCompile(`refresh_mode = '(\w+)'`)
+
 var dynamicTableSchema = map[string]*schema.Schema{
 	"or_replace": {
 		Type:        schema.TypeBool,
@@ -74,6 +79,27 @@ var dynamicTableSchema = map[string]*schema.Schema{
 		Optional:    true,
 		Description: "Specifies a comment for the dynamic table.",
 	},
+	"refresh_mode": {
+		Type:         schema.TypeString,
+		Optional:     true,
+		Default:      sdk.DynamicTableRefreshModeAuto,
+		Description:  "INCREMENTAL to use incremental refreshes, FULL to recompute the whole table on every refresh, or AUTO to let Snowflake decide.",
+		ValidateFunc: validation.StringInSlice(sdk.AsStringList(sdk.AllDynamicRefreshModes), true),
+		ForceNew:     true,
+	},
+	"initialize": {
+		Type:         schema.TypeString,
+		Optional:     true,
+		Default:      sdk.DynamicTableInitializeOnCreate,
+		Description:  "Initialize trigger for the dynamic table. Can only be set on creation.",
+		ValidateFunc: validation.StringInSlice(sdk.AsStringList(sdk.AllDynamicTableInitializes), true),
+		ForceNew:     true,
+	},
+	"created_on": {
+		Type:        schema.TypeString,
+		Description: "Time when this dynamic table was created.",
+		Computed:    true,
+	},
 	"cluster_by": {
 		Type:        schema.TypeString,
 		Description: "The clustering key for the dynamic table.",
@@ -94,11 +120,6 @@ var dynamicTableSchema = map[string]*schema.Schema{
 		Description: "Role that owns the dynamic table.",
 		Computed:    true,
 	},
-	"refresh_mode": {
-		Type:        schema.TypeString,
-		Description: "INCREMENTAL if the dynamic table will use incremental refreshes, or FULL if it will recompute the whole table on every refresh.",
-		Computed:    true,
-	},
 	"refresh_mode_reason": {
 		Type:        schema.TypeString,
 		Description: "Explanation for why FULL refresh mode was chosen. NULL if refresh mode is not FULL.",
@@ -199,6 +220,24 @@ func ReadDynamicTable(d *schema.ResourceData, meta interface{}) error {
 			return err
 		}
 	}
+	if strings.Contains(dynamicTable.Text, "initialize = 'ON_CREATE'") {
+		if err := d.Set("initialize", "ON_CREATE"); err != nil {
+			return err
+		}
+	} else if strings.Contains(dynamicTable.Text, "initialize = 'ON_SCHEDULE'") {
+		if err := d.Set("initialize", "ON_SCHEDULE"); err != nil {
+			return err
+		}
+	}
+	m := refreshModePattern.FindStringSubmatch(dynamicTable.Text)
+	if len(m) > 1 {
+		if err := d.Set("refresh_mode", m[1]); err != nil {
+			return err
+		}
+	}
+	if err := d.Set("created_on", dynamicTable.CreatedOn.Format(time.RFC3339)); err != nil {
+		return err
+	}
 	if err := d.Set("cluster_by", dynamicTable.ClusterBy); err != nil {
 		return err
 	}
@@ -211,9 +250,6 @@ func ReadDynamicTable(d *schema.ResourceData, meta interface{}) error {
 	if err := d.Set("owner", dynamicTable.Owner); err != nil {
 		return err
 	}
-	if err := d.Set("refresh_mode", string(dynamicTable.RefreshMode)); err != nil {
-		return err
-	}
 	if err := d.Set("refresh_mode_reason", dynamicTable.RefreshModeReason); err != nil {
 		return err
 	}
@@ -288,6 +324,12 @@ func CreateDynamicTable(d *schema.ResourceData, meta interface{}) error {
 	if v, ok := d.GetOk("or_replace"); ok && v.(bool) {
 		request.WithOrReplace(true)
 	}
+	if v, ok := d.GetOk("refresh_mode"); ok {
+		request.WithRefreshMode(sdk.DynamicTableRefreshMode(v.(string)))
+	}
+	if v, ok := d.GetOk("initialize"); ok {
+		request.WithInitialize(sdk.DynamicTableInitialize(v.(string)))
+	}
 	if err := client.DynamicTables.Create(context.Background(), request); err != nil {
 		return err
 	}

pkg/resources/dynamic_table_acceptance_test.go

@@ -37,6 +37,16 @@ func TestAcc_DynamicTable_basic(t *testing.T) {
 	variableSet2["warehouse"] = config.StringVariable(acc.TestWarehouseName2)
 	variableSet2["comment"] = config.StringVariable("Terraform acceptance test - updated")
 
+	variableSet3 := m()
+	variableSet3["initialize"] = config.StringVariable(string(sdk.DynamicTableInitializeOnSchedule))
+
+	variableSet4 := m()
+	variableSet4["initialize"] = config.StringVariable(string(sdk.DynamicTableInitializeOnSchedule)) // keep the same setting from set 3
+	variableSet4["refresh_mode"] = config.StringVariable(string(sdk.DynamicTableRefreshModeFull))
+
+	// used to check whether a dynamic table was replaced
+	var createdOn string
+
 	resource.Test(t, resource.TestCase{
 		ProtoV6ProviderFactories: acc.TestAccProtoV6ProviderFactories,
 		PreCheck:                 func() { acc.TestAccPreCheck(t) },
@@ -53,6 +63,8 @@ func TestAcc_DynamicTable_basic(t *testing.T) {
 					resource.TestCheckResourceAttr(resourceName, "database", acc.TestDatabaseName),
 					resource.TestCheckResourceAttr(resourceName, "schema", acc.TestSchemaName),
 					resource.TestCheckResourceAttr(resourceName, "warehouse", acc.TestWarehouseName),
+					resource.TestCheckResourceAttr(resourceName, "initialize", string(sdk.DynamicTableInitializeOnCreate)),
+					resource.TestCheckResourceAttr(resourceName, "refresh_mode", string(sdk.DynamicTableRefreshModeAuto)),
 					resource.TestCheckResourceAttr(resourceName, "target_lag.#", "1"),
 					resource.TestCheckResourceAttr(resourceName, "target_lag.0.maximum_duration", "2 minutes"),
 					resource.TestCheckResourceAttr(resourceName, "query", fmt.Sprintf("select \"id\" from \"%v\".\"%v\".\"%v\"", acc.TestDatabaseName, acc.TestSchemaName, tableName)),
@@ -65,14 +77,18 @@ func TestAcc_DynamicTable_basic(t *testing.T) {
 					resource.TestCheckResourceAttrSet(resourceName, "rows"),
 					resource.TestCheckResourceAttrSet(resourceName, "bytes"),
 					resource.TestCheckResourceAttrSet(resourceName, "owner"),
-					resource.TestCheckResourceAttrSet(resourceName, "refresh_mode"),
 					// - not used at this time
 					// resource.TestCheckResourceAttrSet(resourceName, "automatic_clustering"),
 					resource.TestCheckResourceAttrSet(resourceName, "scheduling_state"),
 					resource.TestCheckResourceAttrSet(resourceName, "last_suspended_on"),
 					resource.TestCheckResourceAttrSet(resourceName, "is_clone"),
 					resource.TestCheckResourceAttrSet(resourceName, "is_replica"),
 					resource.TestCheckResourceAttrSet(resourceName, "data_timestamp"),
+
+					resource.TestCheckResourceAttrWith(resourceName, "created_on", func(value string) error {
+						createdOn = value
+						return nil
+					}),
 				),
 			},
 			// test target lag to downstream and change comment
@@ -88,6 +104,45 @@ func TestAcc_DynamicTable_basic(t *testing.T) {
 					resource.TestCheckResourceAttr(resourceName, "target_lag.#", "1"),
 					resource.TestCheckResourceAttr(resourceName, "target_lag.0.downstream", "true"),
 					resource.TestCheckResourceAttr(resourceName, "comment", "Terraform acceptance test - updated"),
+
+					resource.TestCheckResourceAttrWith(resourceName, "created_on", func(value string) error {
+						if value != createdOn {
+							return fmt.Errorf("created_on changed from %v to %v", createdOn, value)
+						}
+						return nil
+					}),
+				),
+			},
+			// test changing initialize setting
+			{
+				ConfigDirectory: config.TestStepDirectory(),
+				ConfigVariables: variableSet3,
+				Check: resource.ComposeTestCheckFunc(
+					resource.TestCheckResourceAttr(resourceName, "initialize", string(sdk.DynamicTableInitializeOnSchedule)),
+
+					resource.TestCheckResourceAttrWith(resourceName, "created_on", func(value string) error {
+						if value == createdOn {
+							return fmt.Errorf("expected created_on to change but was not changed")
+						}
+						createdOn = value
+						return nil
+					}),
+				),
+			},
+			// test changing refresh_mode setting
+			{
+				ConfigDirectory: config.TestStepDirectory(),
+				ConfigVariables: variableSet4,
+				Check: resource.ComposeTestCheckFunc(
+					resource.TestCheckResourceAttr(resourceName, "initialize", string(sdk.DynamicTableInitializeOnSchedule)),
+					resource.TestCheckResourceAttr(resourceName, "refresh_mode", string(sdk.DynamicTableRefreshModeFull)),
+
+					resource.TestCheckResourceAttrWith(resourceName, "created_on", func(value string) error {
+						if value == createdOn {
+							return fmt.Errorf("expected created_on to change but was not changed")
+						}
+						return nil
+					}),
 				),
 			},
 			// test import

pkg/resources/testdata/TestAcc_DynamicTable_basic/3/test.tf

@@ -19,7 +19,8 @@ resource "snowflake_dynamic_table" "dt" {
   target_lag {
     downstream = true
   }
-  warehouse = var.warehouse
-  query     = var.query
-  comment   = var.comment
+  warehouse    = var.warehouse
+  query        = var.query
+  comment      = var.comment
+  initialize   = var.initialize
 }

pkg/resources/testdata/TestAcc_DynamicTable_basic/3/variables.tf

@@ -24,6 +24,10 @@ variable "comment" {
   type = string
 }
 
+variable "initialize" {
+  type = string
+}
+
 variable "table_name" {
   type = string
 }

pkg/resources/testdata/TestAcc_DynamicTable_basic/4/test.tf

@@ -0,0 +1,27 @@
+
+
+resource "snowflake_table" "t" {
+  database        = var.database
+  schema          = var.schema
+  name            = var.table_name
+  change_tracking = true
+  column {
+    name = "id"
+    type = "NUMBER(38,0)"
+  }
+}
+
+resource "snowflake_dynamic_table" "dt" {
+  depends_on = [snowflake_table.t]
+  name       = var.name
+  database   = var.database
+  schema     = var.schema
+  target_lag {
+    downstream = true
+  }
+  warehouse    = var.warehouse
+  query        = var.query
+  comment      = var.comment
+  refresh_mode = var.refresh_mode
+  initialize   = var.initialize
+}

pkg/resources/testdata/TestAcc_DynamicTable_basic/4/variables.tf

@@ -0,0 +1,37 @@
+
+
+variable "name" {
+  type = string
+}
+
+variable "database" {
+  type = string
+}
+
+variable "schema" {
+  type = string
+}
+
+variable "warehouse" {
+  type = string
+}
+
+variable "query" {
+  type = string
+}
+
+variable "comment" {
+  type = string
+}
+
+variable "refresh_mode" {
+  type = string
+}
+
+variable "initialize" {
+  type = string
+}
+
+variable "table_name" {
+  type = string
+}

pkg/resources/testdata/TestAcc_DynamicTable_basic/5/test.tf

@@ -0,0 +1,25 @@
+
+
+resource "snowflake_table" "t" {
+  database        = var.database
+  schema          = var.schema
+  name            = var.table_name
+  change_tracking = true
+  column {
+    name = "id"
+    type = "NUMBER(38,0)"
+  }
+}
+
+resource "snowflake_dynamic_table" "dt" {
+  depends_on = [snowflake_table.t]
+  name       = var.name
+  database   = var.database
+  schema     = var.schema
+  target_lag {
+    downstream = true
+  }
+  warehouse = var.warehouse
+  query     = var.query
+  comment   = var.comment
+}

pkg/resources/testdata/TestAcc_DynamicTable_basic/5/variables.tf

@@ -0,0 +1,29 @@
+
+
+variable "name" {
+  type = string
+}
+
+variable "database" {
+  type = string
+}
+
+variable "schema" {
+  type = string
+}
+
+variable "warehouse" {
+  type = string
+}
+
+variable "query" {
+  type = string
+}
+
+variable "comment" {
+  type = string
+}
+
+variable "table_name" {
+  type = string
+}