From 3380d20ea14f7247c21bc394c5dd3395dff897e3 Mon Sep 17 00:00:00 2001 From: nikhil Date: Sat, 6 Apr 2024 21:16:11 +0100 Subject: [PATCH 1/5] r/aws_glue_job: update test and documentation --- internal/service/glue/job_test.go | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/internal/service/glue/job_test.go b/internal/service/glue/job_test.go index 0059ce5276a8..cf2cbacc24e9 100644 --- a/internal/service/glue/job_test.go +++ b/internal/service/glue/job_test.go @@ -652,6 +652,34 @@ func TestAccGlueJob_workerType(t *testing.T) { resource.TestCheckResourceAttr(resourceName, "worker_type", "G.2X"), ), }, + { + Config: testAccJobConfig_workerType(rName, "G.025X"), + Check: resource.ComposeTestCheckFunc( + testAccCheckJobExists(ctx, resourceName, &job), + resource.TestCheckResourceAttr(resourceName, "worker_type", "G.025X"), + ), + }, + { + Config: testAccJobConfig_workerType(rName, "G.4X"), + Check: resource.ComposeTestCheckFunc( + testAccCheckJobExists(ctx, resourceName, &job), + resource.TestCheckResourceAttr(resourceName, "worker_type", "G.4X"), + ), + }, + { + Config: testAccJobConfig_workerType(rName, "G.8X"), + Check: resource.ComposeTestCheckFunc( + testAccCheckJobExists(ctx, resourceName, &job), + resource.TestCheckResourceAttr(resourceName, "worker_type", "G.8X"), + ), + }, + { + Config: testAccJobConfig_workerType(rName, "Z.2X"), + Check: resource.ComposeTestCheckFunc( + testAccCheckJobExists(ctx, resourceName, &job), + resource.TestCheckResourceAttr(resourceName, "worker_type", "Z.2X"), + ), + }, { ResourceName: resourceName, ImportState: true, From 2b8e32ef97d70f058d03d18c29a844d58d08eaaf Mon Sep 17 00:00:00 2001 From: nikhil Date: Sat, 6 Apr 2024 21:20:43 +0100 Subject: [PATCH 2/5] r/aws_glue_job: update test and documentation --- website/docs/r/glue_job.html.markdown | 2 ++ 1 file changed, 2 insertions(+) diff --git a/website/docs/r/glue_job.html.markdown b/website/docs/r/glue_job.html.markdown index b6394c1c4c7c..1fd6f3d3f35b 100644 --- a/website/docs/r/glue_job.html.markdown +++ b/website/docs/r/glue_job.html.markdown @@ -121,6 +121,8 @@ This resource supports the following arguments: * For the Standard worker type, each worker provides 4 vCPU, 16 GB of memory and a 50GB disk, and 2 executors per worker. * For the G.1X worker type, each worker maps to 1 DPU (4 vCPU, 16 GB of memory, 64 GB disk), and provides 1 executor per worker. Recommended for memory-intensive jobs. * For the G.2X worker type, each worker maps to 2 DPU (8 vCPU, 32 GB of memory, 128 GB disk), and provides 1 executor per worker. Recommended for memory-intensive jobs. + * For the G.4X worker type, each worker maps to 4 DPU (16 vCPUs, 64 GB of memory) with 256GB disk (approximately 235GB free), and provides 1 executor per worker. Recommended for memory-intensive jobs. Only available for Glue version 3.0. Available AWS Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm). + * For the G.8X worker type, each worker maps to 8 DPU (32 vCPUs, 128 GB of memory) with 512GB disk (approximately 487GB free), and provides 1 executor per worker. Recommended for memory-intensive jobs. Only available for Glue version 3.0. Available AWS Regions: US East (Ohio), US East (N. Virginia), US West (Oregon), Asia Pacific (Singapore), Asia Pacific (Sydney), Asia Pacific (Tokyo), Canada (Central), Europe (Frankfurt), Europe (Ireland), and Europe (Stockholm). * For the G.025X worker type, each worker maps to 0.25 DPU (2 vCPU, 4GB of memory, 64 GB disk), and provides 1 executor per worker. Recommended for low volume streaming jobs. Only available for Glue version 3.0. * For the Z.2X worker type, each worker maps to 2 M-DPU (8vCPU, 64 GB of m emory, 128 GB disk), and provides up to 8 Ray workers based on the autoscaler. * `number_of_workers` - (Optional) The number of workers of a defined workerType that are allocated when a job runs. From a9d38a869704d0852fa8410bf0a70b5f28acad65 Mon Sep 17 00:00:00 2001 From: nikhil Date: Sat, 6 Apr 2024 21:40:46 +0100 Subject: [PATCH 3/5] r/aws_glue_job: update test and documentation --- internal/service/glue/job_test.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/internal/service/glue/job_test.go b/internal/service/glue/job_test.go index cf2cbacc24e9..fa1ffea57909 100644 --- a/internal/service/glue/job_test.go +++ b/internal/service/glue/job_test.go @@ -652,13 +652,6 @@ func TestAccGlueJob_workerType(t *testing.T) { resource.TestCheckResourceAttr(resourceName, "worker_type", "G.2X"), ), }, - { - Config: testAccJobConfig_workerType(rName, "G.025X"), - Check: resource.ComposeTestCheckFunc( - testAccCheckJobExists(ctx, resourceName, &job), - resource.TestCheckResourceAttr(resourceName, "worker_type", "G.025X"), - ), - }, { Config: testAccJobConfig_workerType(rName, "G.4X"), Check: resource.ComposeTestCheckFunc( @@ -673,13 +666,6 @@ func TestAccGlueJob_workerType(t *testing.T) { resource.TestCheckResourceAttr(resourceName, "worker_type", "G.8X"), ), }, - { - Config: testAccJobConfig_workerType(rName, "Z.2X"), - Check: resource.ComposeTestCheckFunc( - testAccCheckJobExists(ctx, resourceName, &job), - resource.TestCheckResourceAttr(resourceName, "worker_type", "Z.2X"), - ), - }, { ResourceName: resourceName, ImportState: true, From 811aef3efbfcd308a812d3acba875d30a902885f Mon Sep 17 00:00:00 2001 From: Jared Baker Date: Tue, 16 Apr 2024 15:49:12 -0400 Subject: [PATCH 4/5] r/aws_glue_job: number_of_workers and worker_type are optional/computed Previously these arguments were optional only, but it appears the read operation can now return non-nil values for these attributes when `max_capacity` is set. Marking these as optional/computed prevents persistent differences (and fixes several failing acceptance tests). ```console % make testacc PKG=glue TESTS=TestAccGlueJob_ ==> Checking that code complies with gofmt requirements... TF_ACC=1 go1.21.8 test ./internal/service/glue/... -v -count 1 -parallel 20 -run='TestAccGlueJob_' -timeout 360m --- PASS: TestAccGlueJob_rayJob (31.89s) --- PASS: TestAccGlueJob_disappears (33.39s) --- PASS: TestAccGlueJob_basicStreaming (36.27s) --- PASS: TestAccGlueJob_basic (38.12s) --- PASS: TestAccGlueJob_nonOverridableArguments (54.61s) --- PASS: TestAccGlueJob_description (54.71s) --- PASS: TestAccGlueJob_defaultArguments (54.78s) --- PASS: TestAccGlueJob_maxCapacity (54.89s) --- PASS: TestAccGlueJob_timeout (54.91s) --- PASS: TestAccGlueJob_command (55.50s) --- PASS: TestAccGlueJob_security (55.58s) --- PASS: TestAccGlueJob_executionClass (55.72s) --- PASS: TestAccGlueJob_streamingTimeout (55.78s) --- PASS: TestAccGlueJob_maxRetries (56.43s) --- PASS: TestAccGlueJob_executionProperty (56.49s) --- PASS: TestAccGlueJob_notificationProperty (57.15s) --- PASS: TestAccGlueJob_tags (63.00s) --- PASS: TestAccGlueJob_glueVersion (63.46s) --- PASS: TestAccGlueJob_pythonShell (73.20s) --- PASS: TestAccGlueJob_workerType (79.32s) PASS ok github.com/hashicorp/terraform-provider-aws/internal/service/glue 84.990s ``` --- internal/service/glue/job.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/service/glue/job.go b/internal/service/glue/job.go index 139ba63a898e..ed3d5a39345c 100644 --- a/internal/service/glue/job.go +++ b/internal/service/glue/job.go @@ -153,6 +153,7 @@ func ResourceJob() *schema.Resource { "number_of_workers": { Type: schema.TypeInt, Optional: true, + Computed: true, ConflictsWith: []string{"max_capacity"}, ValidateFunc: validation.IntAtLeast(1), }, @@ -176,6 +177,7 @@ func ResourceJob() *schema.Resource { "worker_type": { Type: schema.TypeString, Optional: true, + Computed: true, ConflictsWith: []string{"max_capacity"}, ValidateFunc: validation.StringInSlice(glue.WorkerType_Values(), false), }, From 5b4d5570f3df88cd1889e72a540f25238f71bf89 Mon Sep 17 00:00:00 2001 From: Jared Baker Date: Tue, 16 Apr 2024 15:53:09 -0400 Subject: [PATCH 5/5] chore: changelog --- .changelog/36770.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .changelog/36770.txt diff --git a/.changelog/36770.txt b/.changelog/36770.txt new file mode 100644 index 000000000000..1130ca2b82a9 --- /dev/null +++ b/.changelog/36770.txt @@ -0,0 +1,3 @@ +```release-note:bug +resource/aws_glue_job: Mark `number_of_workers` and `worker_type` as optional/computed, preventing persistent differences when `max_capacity` is set. +```