From 21bf4f623a41032661ab8c74aea896f90a735b19 Mon Sep 17 00:00:00 2001 From: DrFaust92 Date: Sun, 23 Aug 2020 22:10:03 +0300 Subject: [PATCH 1/5] support non overridable arguments --- aws/resource_aws_glue_job.go | 32 +++++++++++--- aws/resource_aws_glue_job_test.go | 62 +++++++++++++++++++++++++++ website/docs/r/glue_job.html.markdown | 1 + 3 files changed, 89 insertions(+), 6 deletions(-) diff --git a/aws/resource_aws_glue_job.go b/aws/resource_aws_glue_job.go index ec6a4740058..5ed8cc45735 100644 --- a/aws/resource_aws_glue_job.go +++ b/aws/resource_aws_glue_job.go @@ -137,11 +137,7 @@ func resourceAwsGlueJob() *schema.Resource { Type: schema.TypeString, Optional: true, ConflictsWith: []string{"max_capacity"}, - ValidateFunc: validation.StringInSlice([]string{ - glue.WorkerTypeG1x, - glue.WorkerTypeG2x, - glue.WorkerTypeStandard, - }, false), + ValidateFunc: validation.StringInSlice(glue.WorkerType_Values(), false), }, "number_of_workers": { Type: schema.TypeInt, @@ -149,6 +145,11 @@ func resourceAwsGlueJob() *schema.Resource { ConflictsWith: []string{"max_capacity"}, ValidateFunc: validation.IntAtLeast(2), }, + "non_overridable_arguments": { + Type: schema.TypeMap, + Optional: true, + Elem: &schema.Schema{Type: schema.TypeString}, + }, }, } } @@ -183,6 +184,14 @@ func resourceAwsGlueJobCreate(d *schema.ResourceData, meta interface{}) error { input.DefaultArguments = aws.StringMap(defaultArgumentsMap) } + if kv, ok := d.GetOk("non_overridable_arguments"); ok { + nonOverArgs := make(map[string]string) + for k, v := range kv.(map[string]interface{}) { + nonOverArgs[k] = v.(string) + } + input.NonOverridableArguments = aws.StringMap(nonOverArgs) + } + if v, ok := d.GetOk("description"); ok { input.Description = aws.String(v.(string)) } @@ -270,6 +279,9 @@ func resourceAwsGlueJobRead(d *schema.ResourceData, meta interface{}) error { if err := d.Set("default_arguments", aws.StringValueMap(job.DefaultArguments)); err != nil { return fmt.Errorf("error setting default_arguments: %s", err) } + if err := d.Set("non_overridable_arguments", aws.StringValueMap(job.NonOverridableArguments)); err != nil { + return fmt.Errorf("error setting non_overridable_arguments: %w", err) + } d.Set("description", job.Description) d.Set("glue_version", job.GlueVersion) if err := d.Set("execution_property", flattenGlueExecutionProperty(job.ExecutionProperty)); err != nil { @@ -309,7 +321,7 @@ func resourceAwsGlueJobUpdate(d *schema.ResourceData, meta interface{}) error { if d.HasChanges("command", "connections", "default_arguments", "description", "execution_property", "glue_version", "max_capacity", "max_retries", "notification_property", "number_of_workers", - "role_arn", "security_configuration", "timeout", "worker_type") { + "role_arn", "security_configuration", "timeout", "worker_type", "non_overridable_arguments") { jobUpdate := &glue.JobUpdate{ Command: expandGlueJobCommand(d.Get("command").([]interface{})), Role: aws.String(d.Get("role_arn").(string)), @@ -338,6 +350,14 @@ func resourceAwsGlueJobUpdate(d *schema.ResourceData, meta interface{}) error { jobUpdate.DefaultArguments = aws.StringMap(defaultArgumentsMap) } + if kv, ok := d.GetOk("non_overridable_arguments"); ok { + nonOverArgs := make(map[string]string) + for k, v := range kv.(map[string]interface{}) { + nonOverArgs[k] = v.(string) + } + jobUpdate.NonOverridableArguments = aws.StringMap(nonOverArgs) + } + if v, ok := d.GetOk("description"); ok { jobUpdate.Description = aws.String(v.(string)) } diff --git a/aws/resource_aws_glue_job_test.go b/aws/resource_aws_glue_job_test.go index 2732038e59f..2186dc68a2d 100644 --- a/aws/resource_aws_glue_job_test.go +++ b/aws/resource_aws_glue_job_test.go @@ -75,6 +75,7 @@ func TestAccAWSGlueJob_basic(t *testing.T) { resource.TestCheckResourceAttr(resourceName, "command.#", "1"), resource.TestCheckResourceAttr(resourceName, "command.0.script_location", "testscriptlocation"), resource.TestCheckResourceAttr(resourceName, "default_arguments.%", "0"), + resource.TestCheckResourceAttr(resourceName, "non_overridable_arguments.%", "0"), resource.TestCheckResourceAttr(resourceName, "name", rName), resource.TestCheckResourceAttrPair(resourceName, "role_arn", roleResourceName, "arn"), resource.TestCheckResourceAttr(resourceName, "tags.%", "0"), @@ -164,6 +165,44 @@ func TestAccAWSGlueJob_DefaultArguments(t *testing.T) { }) } +func TestAccAWSGlueJob_nonOverridableArguments(t *testing.T) { + var job glue.Job + + rName := fmt.Sprintf("tf-acc-test-%s", acctest.RandString(5)) + resourceName := "aws_glue_job.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueJobDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAWSGlueJobnonOverridableArgumentsConfig(rName, "job-bookmark-disable", "python"), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueJobExists(resourceName, &job), + resource.TestCheckResourceAttr(resourceName, "non_overridable_arguments.%", "2"), + resource.TestCheckResourceAttr(resourceName, "non_overridable_arguments.--job-bookmark-option", "job-bookmark-disable"), + resource.TestCheckResourceAttr(resourceName, "non_overridable_arguments.--job-language", "python"), + ), + }, + { + Config: testAccAWSGlueJobnonOverridableArgumentsConfig(rName, "job-bookmark-enable", "scala"), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueJobExists(resourceName, &job), + resource.TestCheckResourceAttr(resourceName, "non_overridable_arguments.%", "2"), + resource.TestCheckResourceAttr(resourceName, "non_overridable_arguments.--job-bookmark-option", "job-bookmark-enable"), + resource.TestCheckResourceAttr(resourceName, "non_overridable_arguments.--job-language", "scala"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + }, + }, + }) +} + func TestAccAWSGlueJob_Description(t *testing.T) { var job glue.Job @@ -733,6 +772,29 @@ resource "aws_glue_job" "test" { `, testAccAWSGlueJobConfig_Base(rName), rName, jobBookmarkOption, jobLanguage) } +func testAccAWSGlueJobnonOverridableArgumentsConfig(rName, jobBookmarkOption, jobLanguage string) string { + return fmt.Sprintf(` +%s + +resource "aws_glue_job" "test" { + max_capacity = 10 + name = "%s" + role_arn = aws_iam_role.test.arn + + command { + script_location = "testscriptlocation" + } + + non_overridable_arguments = { + "--job-bookmark-option" = "%s" + "--job-language" = "%s" + } + + depends_on = [aws_iam_role_policy_attachment.test] +} +`, testAccAWSGlueJobConfig_Base(rName), rName, jobBookmarkOption, jobLanguage) +} + func testAccAWSGlueJobConfig_Description(rName, description string) string { return fmt.Sprintf(` %s diff --git a/website/docs/r/glue_job.html.markdown b/website/docs/r/glue_job.html.markdown index 47c9d9e979a..547670ed1e5 100644 --- a/website/docs/r/glue_job.html.markdown +++ b/website/docs/r/glue_job.html.markdown @@ -72,6 +72,7 @@ The following arguments are supported: * `command` – (Required) The command of the job. Defined below. * `connections` – (Optional) The list of connections used for this job. * `default_arguments` – (Optional) The map of default arguments for this job. You can specify arguments here that your own job-execution script consumes, as well as arguments that AWS Glue itself consumes. For information about how to specify and consume your own Job arguments, see the [Calling AWS Glue APIs in Python](http://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-python-calling.html) topic in the developer guide. For information about the key-value pairs that AWS Glue consumes to set up your job, see the [Special Parameters Used by AWS Glue](http://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-python-glue-arguments.html) topic in the developer guide. +* `non_overridable_arguments` – (Optional) Non-overridable arguments for this job, specified as name-value pairs. * `description` – (Optional) Description of the job. * `execution_property` – (Optional) Execution property of the job. Defined below. * `glue_version` - (Optional) The version of glue to use, for example "1.0". For information about available versions, see the [AWS Glue Release Notes](https://docs.aws.amazon.com/glue/latest/dg/release-notes.html). From 2d677a8064a5529038da2c560592c3a0359e7f2c Mon Sep 17 00:00:00 2001 From: DrFaust92 Date: Sun, 23 Aug 2020 22:11:21 +0300 Subject: [PATCH 2/5] add disappears test --- aws/resource_aws_glue_job_test.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/aws/resource_aws_glue_job_test.go b/aws/resource_aws_glue_job_test.go index 2186dc68a2d..4154dfac6e5 100644 --- a/aws/resource_aws_glue_job_test.go +++ b/aws/resource_aws_glue_job_test.go @@ -635,6 +635,29 @@ func TestAccAWSGlueJob_MaxCapacity(t *testing.T) { }) } +func TestAccAWSGlueJob_disappears(t *testing.T) { + var job glue.Job + + rName := fmt.Sprintf("tf-acc-test-%s", acctest.RandString(5)) + resourceName := "aws_glue_job.test" + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSGlueJobDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAWSGlueJobConfig_Required(rName), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSGlueJobExists(resourceName, &job), + testAccCheckResourceDisappears(testAccProvider, resourceAwsGlueJob(), resourceName), + ), + ExpectNonEmptyPlan: true, + }, + }, + }) +} + func testAccCheckAWSGlueJobExists(resourceName string, job *glue.Job) resource.TestCheckFunc { return func(s *terraform.State) error { rs, ok := s.RootModule().Resources[resourceName] From 232c1534fefe4247c3e2b68d71ef24315f461d31 Mon Sep 17 00:00:00 2001 From: Ilia Lazebnik Date: Mon, 24 Aug 2020 10:12:15 +0300 Subject: [PATCH 3/5] Update aws/resource_aws_glue_job.go Co-authored-by: Kit Ewbank --- aws/resource_aws_glue_job.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/aws/resource_aws_glue_job.go b/aws/resource_aws_glue_job.go index 5ed8cc45735..0df8366751e 100644 --- a/aws/resource_aws_glue_job.go +++ b/aws/resource_aws_glue_job.go @@ -185,11 +185,7 @@ func resourceAwsGlueJobCreate(d *schema.ResourceData, meta interface{}) error { } if kv, ok := d.GetOk("non_overridable_arguments"); ok { - nonOverArgs := make(map[string]string) - for k, v := range kv.(map[string]interface{}) { - nonOverArgs[k] = v.(string) - } - input.NonOverridableArguments = aws.StringMap(nonOverArgs) + input.NonOverridableArguments = stringMapToPointers(kv) } if v, ok := d.GetOk("description"); ok { From 2899cf199736e0239d0d858482e7e994e268c47b Mon Sep 17 00:00:00 2001 From: Ilia Lazebnik Date: Mon, 24 Aug 2020 10:12:22 +0300 Subject: [PATCH 4/5] Update aws/resource_aws_glue_job.go Co-authored-by: Kit Ewbank --- aws/resource_aws_glue_job.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/aws/resource_aws_glue_job.go b/aws/resource_aws_glue_job.go index 0df8366751e..4e1aab6a6cc 100644 --- a/aws/resource_aws_glue_job.go +++ b/aws/resource_aws_glue_job.go @@ -347,11 +347,7 @@ func resourceAwsGlueJobUpdate(d *schema.ResourceData, meta interface{}) error { } if kv, ok := d.GetOk("non_overridable_arguments"); ok { - nonOverArgs := make(map[string]string) - for k, v := range kv.(map[string]interface{}) { - nonOverArgs[k] = v.(string) - } - jobUpdate.NonOverridableArguments = aws.StringMap(nonOverArgs) + jobUpdate.NonOverridableArguments = stringMapToPointers(kv) } if v, ok := d.GetOk("description"); ok { From 8da399172b813f39212cc1fa04eaf5d99b712e96 Mon Sep 17 00:00:00 2001 From: DrFaust92 Date: Mon, 24 Aug 2020 10:22:46 +0300 Subject: [PATCH 5/5] use built in `stringMapToPointers` func --- aws/resource_aws_glue_job.go | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/aws/resource_aws_glue_job.go b/aws/resource_aws_glue_job.go index 4e1aab6a6cc..0702af39c8f 100644 --- a/aws/resource_aws_glue_job.go +++ b/aws/resource_aws_glue_job.go @@ -177,15 +177,11 @@ func resourceAwsGlueJobCreate(d *schema.ResourceData, meta interface{}) error { } if kv, ok := d.GetOk("default_arguments"); ok { - defaultArgumentsMap := make(map[string]string) - for k, v := range kv.(map[string]interface{}) { - defaultArgumentsMap[k] = v.(string) - } - input.DefaultArguments = aws.StringMap(defaultArgumentsMap) + input.DefaultArguments = stringMapToPointers(kv.(map[string]interface{})) } if kv, ok := d.GetOk("non_overridable_arguments"); ok { - input.NonOverridableArguments = stringMapToPointers(kv) + input.NonOverridableArguments = stringMapToPointers(kv.(map[string]interface{})) } if v, ok := d.GetOk("description"); ok { @@ -339,15 +335,11 @@ func resourceAwsGlueJobUpdate(d *schema.ResourceData, meta interface{}) error { } if kv, ok := d.GetOk("default_arguments"); ok { - defaultArgumentsMap := make(map[string]string) - for k, v := range kv.(map[string]interface{}) { - defaultArgumentsMap[k] = v.(string) - } - jobUpdate.DefaultArguments = aws.StringMap(defaultArgumentsMap) + jobUpdate.DefaultArguments = stringMapToPointers(kv.(map[string]interface{})) } if kv, ok := d.GetOk("non_overridable_arguments"); ok { - jobUpdate.NonOverridableArguments = stringMapToPointers(kv) + jobUpdate.NonOverridableArguments = stringMapToPointers(kv.(map[string]interface{})) } if v, ok := d.GetOk("description"); ok {