Skip to content

Commit

Permalink
resource/aws_glue_job: Add tags argument and arn attribute
Browse files Browse the repository at this point in the history 
Reference: #7668

Output from acceptance testing:

```
--- PASS: TestAccAWSGlueJob_AllocatedCapacity (34.80s)
--- PASS: TestAccAWSGlueJob_Basic (27.40s)
--- PASS: TestAccAWSGlueJob_Command (35.99s)
--- PASS: TestAccAWSGlueJob_DefaultArguments (39.55s)
--- PASS: TestAccAWSGlueJob_Description (35.79s)
--- PASS: TestAccAWSGlueJob_ExecutionProperty (32.81s)
--- PASS: TestAccAWSGlueJob_GlueVersion (30.32s)
--- PASS: TestAccAWSGlueJob_MaxCapacity (34.52s)
--- PASS: TestAccAWSGlueJob_MaxRetries (30.56s)
--- PASS: TestAccAWSGlueJob_PythonShell (56.96s)
--- PASS: TestAccAWSGlueJob_SecurityConfiguration (39.82s)
--- PASS: TestAccAWSGlueJob_Tags (132.80s)
--- PASS: TestAccAWSGlueJob_Timeout (31.40s)
--- PASS: TestAccAWSGlueJob_WorkerType (57.93s)
```
  • Loading branch information
@bflad
bflad committed Nov 21, 2019
1 parent 3990dfc commit bcca863
Show file tree
Hide file tree
Showing 3 changed files with 191 additions and 49 deletions.
148 changes: 99 additions & 49 deletions aws/resource_aws_glue_job.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,11 @@ import (
"log"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/arn"
"github.com/aws/aws-sdk-go/service/glue"
"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
"github.com/hashicorp/terraform-plugin-sdk/helper/validation"
"github.com/terraform-providers/terraform-provider-aws/aws/internal/keyvaluetags"
)

func resourceAwsGlueJob() *schema.Resource {
Expand All @@ -29,6 +31,10 @@ func resourceAwsGlueJob() *schema.Resource {
Deprecated: "Please use attribute `max_capacity' instead. This attribute might be removed in future releases.",
ValidateFunc: validation.IntAtLeast(2),
},
"arn": {
Type: schema.TypeString,
Computed: true,
},
"command": {
Type: schema.TypeList,
Required: true,
Expand Down Expand Up @@ -109,6 +115,7 @@ func resourceAwsGlueJob() *schema.Resource {
Required: true,
ValidateFunc: validateArn,
},
"tags": tagsSchema(),
"timeout": {
Type: schema.TypeInt,
Optional: true,
Expand Down Expand Up @@ -146,6 +153,7 @@ func resourceAwsGlueJobCreate(d *schema.ResourceData, meta interface{}) error {
Command: expandGlueJobCommand(d.Get("command").([]interface{})),
Name: aws.String(name),
Role: aws.String(d.Get("role_arn").(string)),
Tags: keyvaluetags.New(d.Get("tags").(map[string]interface{})).IgnoreAws().GlueTags(),
Timeout: aws.Int64(int64(d.Get("timeout").(int))),
}

Expand Down Expand Up @@ -236,6 +244,15 @@ func resourceAwsGlueJobRead(d *schema.ResourceData, meta interface{}) error {
return nil
}

jobARN := arn.ARN{
Partition: meta.(*AWSClient).partition,
Service: "glue",
Region: meta.(*AWSClient).region,
AccountID: meta.(*AWSClient).accountid,
Resource: fmt.Sprintf("job/%s", d.Id()),
}.String()
d.Set("arn", jobARN)

if err := d.Set("command", flattenGlueJobCommand(job.Command)); err != nil {
return fmt.Errorf("error setting command: %s", err)
}
Expand All @@ -254,6 +271,17 @@ func resourceAwsGlueJobRead(d *schema.ResourceData, meta interface{}) error {
d.Set("max_retries", int(aws.Int64Value(job.MaxRetries)))
d.Set("name", job.Name)
d.Set("role_arn", job.Role)

tags, err := keyvaluetags.GlueListTags(conn, jobARN)

if err != nil {
return fmt.Errorf("error listing tags for Glue Job (%s): %s", jobARN, err)
}

if err := d.Set("tags", tags.IgnoreAws().Map()); err != nil {
return fmt.Errorf("error setting tags: %s", err)
}

d.Set("timeout", int(aws.Int64Value(job.Timeout)))
if err := d.Set("security_configuration", job.SecurityConfiguration); err != nil {
return fmt.Errorf("error setting security_configuration: %s", err)
Expand All @@ -271,71 +299,93 @@ func resourceAwsGlueJobRead(d *schema.ResourceData, meta interface{}) error {
func resourceAwsGlueJobUpdate(d *schema.ResourceData, meta interface{}) error {
conn := meta.(*AWSClient).glueconn

jobUpdate := &glue.JobUpdate{
Command: expandGlueJobCommand(d.Get("command").([]interface{})),
Role: aws.String(d.Get("role_arn").(string)),
Timeout: aws.Int64(int64(d.Get("timeout").(int))),
}
if d.HasChange("allocated_capacity") ||
d.HasChange("command") ||
d.HasChange("connections") ||
d.HasChange("default_arguments") ||
d.HasChange("description") ||
d.HasChange("execution_property") ||
d.HasChange("glue_version") ||
d.HasChange("max_capacity") ||
d.HasChange("max_retries") ||
d.HasChange("number_of_workers") ||
d.HasChange("role_arn") ||
d.HasChange("security_configuration") ||
d.HasChange("timeout") ||
d.HasChange("worker_type") {
jobUpdate := &glue.JobUpdate{
Command: expandGlueJobCommand(d.Get("command").([]interface{})),
Role: aws.String(d.Get("role_arn").(string)),
Timeout: aws.Int64(int64(d.Get("timeout").(int))),
}

if v, ok := d.GetOk("number_of_workers"); ok {
jobUpdate.NumberOfWorkers = aws.Int64(int64(v.(int)))
} else {
if v, ok := d.GetOk("max_capacity"); ok {
jobUpdate.MaxCapacity = aws.Float64(v.(float64))
if v, ok := d.GetOk("number_of_workers"); ok {
jobUpdate.NumberOfWorkers = aws.Int64(int64(v.(int)))
} else {
if v, ok := d.GetOk("max_capacity"); ok {
jobUpdate.MaxCapacity = aws.Float64(v.(float64))
}
if d.HasChange("allocated_capacity") {
jobUpdate.MaxCapacity = aws.Float64(float64(d.Get("allocated_capacity").(int)))
log.Printf("[WARN] Using deprecated `allocated_capacity' attribute.")
}
}
if d.HasChange("allocated_capacity") {
jobUpdate.MaxCapacity = aws.Float64(float64(d.Get("allocated_capacity").(int)))
log.Printf("[WARN] Using deprecated `allocated_capacity' attribute.")

if v, ok := d.GetOk("connections"); ok {
jobUpdate.Connections = &glue.ConnectionsList{
Connections: expandStringList(v.([]interface{})),
}
}
}

if v, ok := d.GetOk("connections"); ok {
jobUpdate.Connections = &glue.ConnectionsList{
Connections: expandStringList(v.([]interface{})),
if kv, ok := d.GetOk("default_arguments"); ok {
defaultArgumentsMap := make(map[string]string)
for k, v := range kv.(map[string]interface{}) {
defaultArgumentsMap[k] = v.(string)
}
jobUpdate.DefaultArguments = aws.StringMap(defaultArgumentsMap)
}
}

if kv, ok := d.GetOk("default_arguments"); ok {
defaultArgumentsMap := make(map[string]string)
for k, v := range kv.(map[string]interface{}) {
defaultArgumentsMap[k] = v.(string)
if v, ok := d.GetOk("description"); ok {
jobUpdate.Description = aws.String(v.(string))
}
jobUpdate.DefaultArguments = aws.StringMap(defaultArgumentsMap)
}

if v, ok := d.GetOk("description"); ok {
jobUpdate.Description = aws.String(v.(string))
}
if v, ok := d.GetOk("glue_version"); ok {
jobUpdate.GlueVersion = aws.String(v.(string))
}

if v, ok := d.GetOk("glue_version"); ok {
jobUpdate.GlueVersion = aws.String(v.(string))
}
if v, ok := d.GetOk("execution_property"); ok {
jobUpdate.ExecutionProperty = expandGlueExecutionProperty(v.([]interface{}))
}

if v, ok := d.GetOk("execution_property"); ok {
jobUpdate.ExecutionProperty = expandGlueExecutionProperty(v.([]interface{}))
}
if v, ok := d.GetOk("max_retries"); ok {
jobUpdate.MaxRetries = aws.Int64(int64(v.(int)))
}

if v, ok := d.GetOk("max_retries"); ok {
jobUpdate.MaxRetries = aws.Int64(int64(v.(int)))
}
if v, ok := d.GetOk("security_configuration"); ok {
jobUpdate.SecurityConfiguration = aws.String(v.(string))
}

if v, ok := d.GetOk("security_configuration"); ok {
jobUpdate.SecurityConfiguration = aws.String(v.(string))
}
if v, ok := d.GetOk("worker_type"); ok {
jobUpdate.WorkerType = aws.String(v.(string))
}

if v, ok := d.GetOk("worker_type"); ok {
jobUpdate.WorkerType = aws.String(v.(string))
}
input := &glue.UpdateJobInput{
JobName: aws.String(d.Id()),
JobUpdate: jobUpdate,
}

input := &glue.UpdateJobInput{
JobName: aws.String(d.Id()),
JobUpdate: jobUpdate,
log.Printf("[DEBUG] Updating Glue Job: %s", input)
_, err := conn.UpdateJob(input)
if err != nil {
return fmt.Errorf("error updating Glue Job (%s): %s", d.Id(), err)
}
}

log.Printf("[DEBUG] Updating Glue Job: %s", input)
_, err := conn.UpdateJob(input)
if err != nil {
return fmt.Errorf("error updating Glue Job (%s): %s", d.Id(), err)
if d.HasChange("tags") {
o, n := d.GetChange("tags")
if err := keyvaluetags.GlueUpdateTags(conn, d.Get("arn").(string), o, n); err != nil {
return fmt.Errorf("error updating tags: %s", err)
}
}

return resourceAwsGlueJobRead(d, meta)
Expand Down
90 changes: 90 additions & 0 deletions aws/resource_aws_glue_job_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,11 +70,13 @@ func TestAccAWSGlueJob_Basic(t *testing.T) {
Config: testAccAWSGlueJobConfig_Required(rName),
Check: resource.ComposeTestCheckFunc(
testAccCheckAWSGlueJobExists(resourceName, &job),
testAccCheckResourceAttrRegionalARN(resourceName, "arn", "glue", fmt.Sprintf("job/%s", rName)),
resource.TestCheckResourceAttr(resourceName, "command.#", "1"),
resource.TestCheckResourceAttr(resourceName, "command.0.script_location", "testscriptlocation"),
resource.TestCheckResourceAttr(resourceName, "default_arguments.%", "0"),
resource.TestCheckResourceAttr(resourceName, "name", rName),
resource.TestMatchResourceAttr(resourceName, "role_arn", regexp.MustCompile(fmt.Sprintf("^arn:[^:]+:iam::[^:]+:role/%s", rName))),
resource.TestCheckResourceAttr(resourceName, "tags.%", "0"),
resource.TestCheckResourceAttr(resourceName, "timeout", "2880"),
),
},
Expand Down Expand Up @@ -345,6 +347,51 @@ func TestAccAWSGlueJob_MaxRetries(t *testing.T) {
})
}

func TestAccAWSGlueJob_Tags(t *testing.T) {
var job1, job2, job3 glue.Job

rName := fmt.Sprintf("tf-acc-test-%s", acctest.RandString(5))
resourceName := "aws_glue_job.test"

resource.ParallelTest(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
Providers: testAccProviders,
CheckDestroy: testAccCheckAWSGlueJobDestroy,
Steps: []resource.TestStep{
{
Config: testAccAWSGlueJobConfigTags1(rName, "key1", "value1"),
Check: resource.ComposeTestCheckFunc(
testAccCheckAWSGlueJobExists(resourceName, &job1),
resource.TestCheckResourceAttr(resourceName, "tags.%", "1"),
resource.TestCheckResourceAttr(resourceName, "tags.key1", "value1"),
),
},
{
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
},
{
Config: testAccAWSGlueJobConfigTags2(rName, "key1", "value1updated", "key2", "value2"),
Check: resource.ComposeTestCheckFunc(
testAccCheckAWSGlueJobExists(resourceName, &job2),
resource.TestCheckResourceAttr(resourceName, "tags.%", "2"),
resource.TestCheckResourceAttr(resourceName, "tags.key1", "value1updated"),
resource.TestCheckResourceAttr(resourceName, "tags.key2", "value2"),
),
},
{
Config: testAccAWSGlueJobConfigTags1(rName, "key2", "value2"),
Check: resource.ComposeTestCheckFunc(
testAccCheckAWSGlueJobExists(resourceName, &job3),
resource.TestCheckResourceAttr(resourceName, "tags.%", "1"),
resource.TestCheckResourceAttr(resourceName, "tags.key2", "value2"),
),
},
},
})
}

func TestAccAWSGlueJob_Timeout(t *testing.T) {
var job glue.Job

Expand Down Expand Up @@ -798,6 +845,49 @@ resource "aws_glue_job" "test" {
`, testAccAWSGlueJobConfig_Base(rName), rName)
}

func testAccAWSGlueJobConfigTags1(rName, tagKey1, tagValue1 string) string {
return testAccAWSGlueJobConfig_Base(rName) + fmt.Sprintf(`
resource "aws_glue_job" "test" {
name = %[1]q
number_of_workers = 2
role_arn = "${aws_iam_role.test.arn}"
worker_type = "Standard"
command {
script_location = "testscriptlocation"
}
tags = {
%[2]q = %[3]q
}
depends_on = ["aws_iam_role_policy_attachment.test"]
}
`, rName, tagKey1, tagValue1)
}

func testAccAWSGlueJobConfigTags2(rName, tagKey1, tagValue1, tagKey2, tagValue2 string) string {
return testAccAWSGlueJobConfig_Base(rName) + fmt.Sprintf(`
resource "aws_glue_job" "test" {
name = %[1]q
number_of_workers = 2
role_arn = "${aws_iam_role.test.arn}"
worker_type = "Standard"
command {
script_location = "testscriptlocation"
}
tags = {
%[2]q = %[3]q
%[4]q = %[5]q
}
depends_on = ["aws_iam_role_policy_attachment.test"]
}
`, rName, tagKey1, tagValue1, tagKey2, tagValue2)
}

func testAccAWSGlueJobConfig_Timeout(rName string, timeout int) string {
return fmt.Sprintf(`
%s
Expand Down
2 changes: 2 additions & 0 deletions website/docs/r/glue_job.html.markdown
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ be removed in future releases, please use `max_capacity` instead.
* `max_retries` – (Optional) The maximum number of times to retry this job if it fails.
* `name` – (Required) The name you assign to this job. It must be unique in your account.
* `role_arn` – (Required) The ARN of the IAM role associated with this job.
* `tags` - (Optional) Key-value mapping of resource tags
* `timeout` – (Optional) The job timeout in minutes. The default is 2880 minutes (48 hours).
* `security_configuration` - (Optional) The name of the Security Configuration to be associated with the job.
* `worker_type` - (Optional) The type of predefined worker that is allocated when a job runs. Accepts a value of Standard, G.1X, or G.2X.
Expand All @@ -79,6 +80,7 @@ be removed in future releases, please use `max_capacity` instead.

In addition to all arguments above, the following attributes are exported:

* `arn` - Amazon Resource Name (ARN) of Glue Job
* `id` - Job name

## Import
Expand Down

0 comments on commit bcca863

Please sign in to comment.