From 56eaf42ddff81218c145976401c4bc6e43b44225 Mon Sep 17 00:00:00 2001 From: The Magician Date: Mon, 26 Oct 2020 14:22:00 -0700 Subject: [PATCH] Add scheduleOptions to BigQuery Data Transfer (#3895) (#2641) * Add scheduleOptions to BigQuery Data Transfer * Fix typo * Update set the timezone to UTC * Update API type to Time Signed-off-by: Modular Magician --- .changelog/3895.txt | 3 + .../resource_bigquery_data_transfer_config.go | 132 ++++++++++++++++++ ...urce_bigquery_data_transfer_config_test.go | 24 +++- ...igquery_data_transfer_config.html.markdown | 30 ++++ 4 files changed, 184 insertions(+), 5 deletions(-) create mode 100644 .changelog/3895.txt diff --git a/.changelog/3895.txt b/.changelog/3895.txt new file mode 100644 index 0000000000..b1548afbdb --- /dev/null +++ b/.changelog/3895.txt @@ -0,0 +1,3 @@ +```release-note:enhancement +bigquery: added `schedule_options` field to `google_bigquery_data_transfer_config` resource +``` diff --git a/google-beta/resource_bigquery_data_transfer_config.go b/google-beta/resource_bigquery_data_transfer_config.go index 900165ef02..f7f11519b7 100644 --- a/google-beta/resource_bigquery_data_transfer_config.go +++ b/google-beta/resource_bigquery_data_transfer_config.go @@ -122,6 +122,45 @@ about the format here: https://cloud.google.com/appengine/docs/flexible/python/scheduling-jobs-with-cron-yaml#the_schedule_format NOTE: the granularity should be at least 8 hours, or less frequent.`, }, + "schedule_options": { + Type: schema.TypeList, + Optional: true, + Description: `Options customizing the data transfer schedule.`, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "disable_auto_scheduling": { + Type: schema.TypeBool, + Optional: true, + Description: `If true, automatic scheduling of data transfer runs for this +configuration will be disabled. The runs can be started on ad-hoc +basis using transferConfigs.startManualRuns API. When automatic +scheduling is disabled, the TransferConfig.schedule field will +be ignored.`, + AtLeastOneOf: []string{"schedule_options.0.disable_auto_scheduling", "schedule_options.0.start_time", "schedule_options.0.end_time"}, + }, + "end_time": { + Type: schema.TypeString, + Optional: true, + Description: `Defines time to stop scheduling transfer runs. A transfer run cannot be +scheduled at or after the end time. The end time can be changed at any +moment. The time when a data transfer can be triggered manually is not +limited by this option.`, + AtLeastOneOf: []string{"schedule_options.0.disable_auto_scheduling", "schedule_options.0.start_time", "schedule_options.0.end_time"}, + }, + "start_time": { + Type: schema.TypeString, + Optional: true, + Description: `Specifies time to start scheduling transfer runs. The first run will be +scheduled at or after the start time according to a recurrence pattern +defined in the schedule string. The start time can be changed at any +moment. The time when a data transfer can be triggered manually is not +limited by this option.`, + AtLeastOneOf: []string{"schedule_options.0.disable_auto_scheduling", "schedule_options.0.start_time", "schedule_options.0.end_time"}, + }, + }, + }, + }, "sensitive_params": { Type: schema.TypeList, Optional: true, @@ -203,6 +242,12 @@ func resourceBigqueryDataTransferConfigCreate(d *schema.ResourceData, meta inter } else if v, ok := d.GetOkExists("schedule"); !isEmptyValue(reflect.ValueOf(scheduleProp)) && (ok || !reflect.DeepEqual(v, scheduleProp)) { obj["schedule"] = scheduleProp } + scheduleOptionsProp, err := expandBigqueryDataTransferConfigScheduleOptions(d.Get("schedule_options"), d, config) + if err != nil { + return err + } else if v, ok := d.GetOkExists("schedule_options"); !isEmptyValue(reflect.ValueOf(scheduleOptionsProp)) && (ok || !reflect.DeepEqual(v, scheduleOptionsProp)) { + obj["scheduleOptions"] = scheduleOptionsProp + } notificationPubsubTopicProp, err := expandBigqueryDataTransferConfigNotificationPubsubTopic(d.Get("notification_pubsub_topic"), d, config) if err != nil { return err @@ -351,6 +396,9 @@ func resourceBigqueryDataTransferConfigRead(d *schema.ResourceData, meta interfa if err := d.Set("schedule", flattenBigqueryDataTransferConfigSchedule(res["schedule"], d, config)); err != nil { return fmt.Errorf("Error reading Config: %s", err) } + if err := d.Set("schedule_options", flattenBigqueryDataTransferConfigScheduleOptions(res["scheduleOptions"], d, config)); err != nil { + return fmt.Errorf("Error reading Config: %s", err) + } if err := d.Set("notification_pubsub_topic", flattenBigqueryDataTransferConfigNotificationPubsubTopic(res["notificationPubsubTopic"], d, config)); err != nil { return fmt.Errorf("Error reading Config: %s", err) } @@ -395,6 +443,12 @@ func resourceBigqueryDataTransferConfigUpdate(d *schema.ResourceData, meta inter } else if v, ok := d.GetOkExists("schedule"); !isEmptyValue(reflect.ValueOf(v)) && (ok || !reflect.DeepEqual(v, scheduleProp)) { obj["schedule"] = scheduleProp } + scheduleOptionsProp, err := expandBigqueryDataTransferConfigScheduleOptions(d.Get("schedule_options"), d, config) + if err != nil { + return err + } else if v, ok := d.GetOkExists("schedule_options"); !isEmptyValue(reflect.ValueOf(v)) && (ok || !reflect.DeepEqual(v, scheduleOptionsProp)) { + obj["scheduleOptions"] = scheduleOptionsProp + } notificationPubsubTopicProp, err := expandBigqueryDataTransferConfigNotificationPubsubTopic(d.Get("notification_pubsub_topic"), d, config) if err != nil { return err @@ -441,6 +495,10 @@ func resourceBigqueryDataTransferConfigUpdate(d *schema.ResourceData, meta inter updateMask = append(updateMask, "schedule") } + if d.HasChange("schedule_options") { + updateMask = append(updateMask, "scheduleOptions") + } + if d.HasChange("notification_pubsub_topic") { updateMask = append(updateMask, "notificationPubsubTopic") } @@ -548,6 +606,35 @@ func flattenBigqueryDataTransferConfigSchedule(v interface{}, d *schema.Resource return v } +func flattenBigqueryDataTransferConfigScheduleOptions(v interface{}, d *schema.ResourceData, config *Config) interface{} { + if v == nil { + return nil + } + original := v.(map[string]interface{}) + if len(original) == 0 { + return nil + } + transformed := make(map[string]interface{}) + transformed["disable_auto_scheduling"] = + flattenBigqueryDataTransferConfigScheduleOptionsDisableAutoScheduling(original["disableAutoScheduling"], d, config) + transformed["start_time"] = + flattenBigqueryDataTransferConfigScheduleOptionsStartTime(original["startTime"], d, config) + transformed["end_time"] = + flattenBigqueryDataTransferConfigScheduleOptionsEndTime(original["endTime"], d, config) + return []interface{}{transformed} +} +func flattenBigqueryDataTransferConfigScheduleOptionsDisableAutoScheduling(v interface{}, d *schema.ResourceData, config *Config) interface{} { + return v +} + +func flattenBigqueryDataTransferConfigScheduleOptionsStartTime(v interface{}, d *schema.ResourceData, config *Config) interface{} { + return v +} + +func flattenBigqueryDataTransferConfigScheduleOptionsEndTime(v interface{}, d *schema.ResourceData, config *Config) interface{} { + return v +} + func flattenBigqueryDataTransferConfigNotificationPubsubTopic(v interface{}, d *schema.ResourceData, config *Config) interface{} { return v } @@ -603,6 +690,51 @@ func expandBigqueryDataTransferConfigSchedule(v interface{}, d TerraformResource return v, nil } +func expandBigqueryDataTransferConfigScheduleOptions(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) { + l := v.([]interface{}) + if len(l) == 0 || l[0] == nil { + return nil, nil + } + raw := l[0] + original := raw.(map[string]interface{}) + transformed := make(map[string]interface{}) + + transformedDisableAutoScheduling, err := expandBigqueryDataTransferConfigScheduleOptionsDisableAutoScheduling(original["disable_auto_scheduling"], d, config) + if err != nil { + return nil, err + } else if val := reflect.ValueOf(transformedDisableAutoScheduling); val.IsValid() && !isEmptyValue(val) { + transformed["disableAutoScheduling"] = transformedDisableAutoScheduling + } + + transformedStartTime, err := expandBigqueryDataTransferConfigScheduleOptionsStartTime(original["start_time"], d, config) + if err != nil { + return nil, err + } else if val := reflect.ValueOf(transformedStartTime); val.IsValid() && !isEmptyValue(val) { + transformed["startTime"] = transformedStartTime + } + + transformedEndTime, err := expandBigqueryDataTransferConfigScheduleOptionsEndTime(original["end_time"], d, config) + if err != nil { + return nil, err + } else if val := reflect.ValueOf(transformedEndTime); val.IsValid() && !isEmptyValue(val) { + transformed["endTime"] = transformedEndTime + } + + return transformed, nil +} + +func expandBigqueryDataTransferConfigScheduleOptionsDisableAutoScheduling(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) { + return v, nil +} + +func expandBigqueryDataTransferConfigScheduleOptionsStartTime(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) { + return v, nil +} + +func expandBigqueryDataTransferConfigScheduleOptionsEndTime(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) { + return v, nil +} + func expandBigqueryDataTransferConfigNotificationPubsubTopic(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) { return v, nil } diff --git a/google-beta/resource_bigquery_data_transfer_config_test.go b/google-beta/resource_bigquery_data_transfer_config_test.go index b6f009ba77..80e3cb75cd 100644 --- a/google-beta/resource_bigquery_data_transfer_config_test.go +++ b/google-beta/resource_bigquery_data_transfer_config_test.go @@ -4,6 +4,7 @@ import ( "fmt" "strings" "testing" + "time" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" @@ -33,6 +34,9 @@ func TestAccBigqueryDataTransferConfig(t *testing.T) { func testAccBigqueryDataTransferConfig_scheduledQuery_basic(t *testing.T) { random_suffix := randString(t, 10) + now := time.Now().UTC() + start_time := now.Add(1 * time.Hour).Format(time.RFC3339) + end_time := now.AddDate(0, 1, 0).Format(time.RFC3339) vcrTest(t, resource.TestCase{ PreCheck: func() { testAccPreCheck(t) }, @@ -40,7 +44,7 @@ func testAccBigqueryDataTransferConfig_scheduledQuery_basic(t *testing.T) { CheckDestroy: testAccCheckBigqueryDataTransferConfigDestroyProducer(t), Steps: []resource.TestStep{ { - Config: testAccBigqueryDataTransferConfig_scheduledQuery(random_suffix, "third", "y"), + Config: testAccBigqueryDataTransferConfig_scheduledQuery(random_suffix, "third", start_time, end_time, "y"), }, { ResourceName: "google_bigquery_data_transfer_config.query_config", @@ -54,6 +58,11 @@ func testAccBigqueryDataTransferConfig_scheduledQuery_basic(t *testing.T) { func testAccBigqueryDataTransferConfig_scheduledQuery_update(t *testing.T) { random_suffix := randString(t, 10) + now := time.Now().UTC() + first_start_time := now.Add(1 * time.Hour).Format(time.RFC3339) + first_end_time := now.AddDate(0, 1, 0).Format(time.RFC3339) + second_start_time := now.Add(2 * time.Hour).Format(time.RFC3339) + second_end_time := now.AddDate(0, 2, 0).Format(time.RFC3339) vcrTest(t, resource.TestCase{ PreCheck: func() { testAccPreCheck(t) }, @@ -61,10 +70,10 @@ func testAccBigqueryDataTransferConfig_scheduledQuery_update(t *testing.T) { CheckDestroy: testAccCheckBigqueryDataTransferConfigDestroyProducer(t), Steps: []resource.TestStep{ { - Config: testAccBigqueryDataTransferConfig_scheduledQuery(random_suffix, "first", "y"), + Config: testAccBigqueryDataTransferConfig_scheduledQuery(random_suffix, "first", first_start_time, first_end_time, "y"), }, { - Config: testAccBigqueryDataTransferConfig_scheduledQuery(random_suffix, "second", "z"), + Config: testAccBigqueryDataTransferConfig_scheduledQuery(random_suffix, "second", second_start_time, second_end_time, "z"), }, { ResourceName: "google_bigquery_data_transfer_config.query_config", @@ -145,7 +154,7 @@ func testAccCheckBigqueryDataTransferConfigDestroyProducer(t *testing.T) func(s } } -func testAccBigqueryDataTransferConfig_scheduledQuery(random_suffix, schedule, letter string) string { +func testAccBigqueryDataTransferConfig_scheduledQuery(random_suffix, schedule, start_time, end_time, letter string) string { return fmt.Sprintf(` data "google_project" "project" {} @@ -175,6 +184,11 @@ resource "google_bigquery_data_transfer_config" "query_config" { location = "asia-northeast1" data_source_id = "scheduled_query" schedule = "%s sunday of quarter 00:00" + schedule_options { + disable_auto_scheduling = false + start_time = "%s" + end_time = "%s" + } destination_dataset_id = google_bigquery_dataset.my_dataset.dataset_id notification_pubsub_topic = google_pubsub_topic.my_topic.id params = { @@ -183,7 +197,7 @@ resource "google_bigquery_data_transfer_config" "query_config" { query = "SELECT name FROM tabl WHERE x = '%s'" } } -`, random_suffix, random_suffix, random_suffix, schedule, letter) +`, random_suffix, random_suffix, random_suffix, schedule, start_time, end_time, letter) } func testAccBigqueryDataTransferConfig_scheduledQuery_service_account(random_suffix string) string { diff --git a/website/docs/r/bigquery_data_transfer_config.html.markdown b/website/docs/r/bigquery_data_transfer_config.html.markdown index 9fdefbc621..a721b15ebb 100644 --- a/website/docs/r/bigquery_data_transfer_config.html.markdown +++ b/website/docs/r/bigquery_data_transfer_config.html.markdown @@ -108,6 +108,11 @@ The following arguments are supported: https://cloud.google.com/appengine/docs/flexible/python/scheduling-jobs-with-cron-yaml#the_schedule_format NOTE: the granularity should be at least 8 hours, or less frequent. +* `schedule_options` - + (Optional) + Options customizing the data transfer schedule. + Structure is documented below. + * `notification_pubsub_topic` - (Optional) Pub/Sub topic where notifications will be sent after transfer runs @@ -150,6 +155,31 @@ The following arguments are supported: If it is not provided, the provider project is used. +The `schedule_options` block supports: + +* `disable_auto_scheduling` - + (Optional) + If true, automatic scheduling of data transfer runs for this + configuration will be disabled. The runs can be started on ad-hoc + basis using transferConfigs.startManualRuns API. When automatic + scheduling is disabled, the TransferConfig.schedule field will + be ignored. + +* `start_time` - + (Optional) + Specifies time to start scheduling transfer runs. The first run will be + scheduled at or after the start time according to a recurrence pattern + defined in the schedule string. The start time can be changed at any + moment. The time when a data transfer can be triggered manually is not + limited by this option. + +* `end_time` - + (Optional) + Defines time to stop scheduling transfer runs. A transfer run cannot be + scheduled at or after the end time. The end time can be changed at any + moment. The time when a data transfer can be triggered manually is not + limited by this option. + The `sensitive_params` block supports: * `secret_access_key` -