Skip to content

Commit

Permalink
Merge branch 'f-glue-workflow-improvements'
Browse files Browse the repository at this point in the history
  • Loading branch information
@bflad
bflad committed Nov 19, 2019
2 parents cb948d3 + 4d2b505 commit 5c9e5e0
Show file tree
Hide file tree
Showing 8 changed files with 733 additions and 17 deletions.
1 change: 1 addition & 0 deletions aws/provider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -540,6 +540,7 @@ func Provider() terraform.ResourceProvider {
"aws_glue_job": resourceAwsGlueJob(),
"aws_glue_security_configuration": resourceAwsGlueSecurityConfiguration(),
"aws_glue_trigger": resourceAwsGlueTrigger(),
"aws_glue_workflow": resourceAwsGlueWorkflow(),
"aws_guardduty_detector": resourceAwsGuardDutyDetector(),
"aws_guardduty_invite_accepter": resourceAwsGuardDutyInviteAccepter(),
"aws_guardduty_ipset": resourceAwsGuardDutyIpset(),
Expand Down
99 changes: 88 additions & 11 deletions aws/resource_aws_glue_trigger.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,13 @@ func resourceAwsGlueTrigger() *schema.Resource {
Type: schema.TypeMap,
Optional: true,
},
"crawler_name": {
Type: schema.TypeString,
Optional: true,
},
"job_name": {
Type: schema.TypeString,
Required: true,
Optional: true,
},
"timeout": {
Type: schema.TypeInt,
Expand Down Expand Up @@ -77,7 +81,11 @@ func resourceAwsGlueTrigger() *schema.Resource {
Schema: map[string]*schema.Schema{
"job_name": {
Type: schema.TypeString,
Required: true,
Optional: true,
},
"crawler_name": {
Type: schema.TypeString,
Optional: true,
},
"logical_operator": {
Type: schema.TypeString,
Expand All @@ -88,15 +96,27 @@ func resourceAwsGlueTrigger() *schema.Resource {
}, false),
},
"state": {
Type: schema.TypeString,
Required: true,
Type: schema.TypeString,
Optional: true,
ConflictsWith: []string{"predicate.0.conditions.0.crawl_state"},
ValidateFunc: validation.StringInSlice([]string{
glue.JobRunStateFailed,
glue.JobRunStateStopped,
glue.JobRunStateSucceeded,
glue.JobRunStateTimeout,
}, false),
},
"crawl_state": {
Type: schema.TypeString,
Optional: true,
ConflictsWith: []string{"predicate.0.conditions.0.state"},
ValidateFunc: validation.StringInSlice([]string{
glue.CrawlStateRunning,
glue.CrawlStateSucceeded,
glue.CrawlStateCancelled,
glue.CrawlStateFailed,
}, false),
},
},
},
},
Expand Down Expand Up @@ -126,6 +146,11 @@ func resourceAwsGlueTrigger() *schema.Resource {
glue.TriggerTypeScheduled,
}, false),
},
"workflow_name": {
Type: schema.TypeString,
Optional: true,
ForceNew: true,
},
},
}
}
Expand Down Expand Up @@ -153,10 +178,18 @@ func resourceAwsGlueTriggerCreate(d *schema.ResourceData, meta interface{}) erro
input.Schedule = aws.String(v.(string))
}

if v, ok := d.GetOk("workflow_name"); ok {
input.WorkflowName = aws.String(v.(string))
}

if d.Get("enabled").(bool) && triggerType != glue.TriggerTypeOnDemand {
input.StartOnCreation = aws.Bool(true)
}

if v, ok := d.GetOk("workflow_name"); ok {
input.WorkflowName = aws.String(v.(string))
}

log.Printf("[DEBUG] Creating Glue Trigger: %s", input)
_, err := conn.CreateTrigger(input)
if err != nil {
Expand Down Expand Up @@ -234,6 +267,7 @@ func resourceAwsGlueTriggerRead(d *schema.ResourceData, meta interface{}) error
d.Set("name", trigger.Name)
d.Set("schedule", trigger.Schedule)
d.Set("type", trigger.Type)
d.Set("workflow_name", trigger.WorkflowName)

return nil
}
Expand Down Expand Up @@ -365,8 +399,14 @@ func expandGlueActions(l []interface{}) []*glue.Action {
for _, mRaw := range l {
m := mRaw.(map[string]interface{})

action := &glue.Action{
JobName: aws.String(m["job_name"].(string)),
action := &glue.Action{}

if v, ok := m["crawler_name"].(string); ok && v != "" {
action.CrawlerName = aws.String(v)
}

if v, ok := m["job_name"].(string); ok && v != "" {
action.JobName = aws.String(v)
}

argumentsMap := make(map[string]string)
Expand All @@ -392,9 +432,23 @@ func expandGlueConditions(l []interface{}) []*glue.Condition {
m := mRaw.(map[string]interface{})

condition := &glue.Condition{
JobName: aws.String(m["job_name"].(string)),
LogicalOperator: aws.String(m["logical_operator"].(string)),
State: aws.String(m["state"].(string)),
}

if v, ok := m["crawler_name"].(string); ok && v != "" {
condition.CrawlerName = aws.String(v)
}

if v, ok := m["crawl_state"].(string); ok && v != "" {
condition.CrawlState = aws.String(v)
}

if v, ok := m["job_name"].(string); ok && v != "" {
condition.JobName = aws.String(v)
}

if v, ok := m["state"].(string); ok && v != "" {
condition.State = aws.String(v)
}

conditions = append(conditions, condition)
Expand Down Expand Up @@ -423,9 +477,17 @@ func flattenGlueActions(actions []*glue.Action) []interface{} {
for _, action := range actions {
m := map[string]interface{}{
"arguments": aws.StringValueMap(action.Arguments),
"job_name": aws.StringValue(action.JobName),
"timeout": int(aws.Int64Value(action.Timeout)),
}

if v := aws.StringValue(action.CrawlerName); v != "" {
m["crawler_name"] = v
}

if v := aws.StringValue(action.JobName); v != "" {
m["job_name"] = v
}

l = append(l, m)
}

Expand All @@ -437,10 +499,25 @@ func flattenGlueConditions(conditions []*glue.Condition) []interface{} {

for _, condition := range conditions {
m := map[string]interface{}{
"job_name": aws.StringValue(condition.JobName),
"logical_operator": aws.StringValue(condition.LogicalOperator),
"state": aws.StringValue(condition.State),
}

if v := aws.StringValue(condition.CrawlerName); v != "" {
m["crawler_name"] = v
}

if v := aws.StringValue(condition.CrawlState); v != "" {
m["crawl_state"] = v
}

if v := aws.StringValue(condition.JobName); v != "" {
m["job_name"] = v
}

if v := aws.StringValue(condition.State); v != "" {
m["state"] = v
}

l = append(l, m)
}

Expand Down
130 changes: 129 additions & 1 deletion aws/resource_aws_glue_trigger_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ func testSweepGlueTriggers(region string) error {
name := aws.StringValue(trigger.Name)

log.Printf("[INFO] Deleting Glue Trigger: %s", name)
err := deleteGlueJob(conn, name)
err := deleteGlueTrigger(conn, name)
if err != nil {
log.Printf("[ERROR] Failed to delete Glue Trigger %s: %s", name, err)
}
Expand Down Expand Up @@ -77,6 +77,53 @@ func TestAccAWSGlueTrigger_Basic(t *testing.T) {
resource.TestCheckResourceAttr(resourceName, "predicate.#", "0"),
resource.TestCheckResourceAttr(resourceName, "schedule", ""),
resource.TestCheckResourceAttr(resourceName, "type", "ON_DEMAND"),
resource.TestCheckResourceAttr(resourceName, "workflow_name", ""),
),
},
{
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
},
},
})
}

func TestAccAWSGlueTrigger_Crawler(t *testing.T) {
var trigger glue.Trigger

rName := fmt.Sprintf("tf-acc-test-%s", acctest.RandString(5))
resourceName := "aws_glue_trigger.test_trigger"

resource.ParallelTest(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
Providers: testAccProviders,
CheckDestroy: testAccCheckAWSGlueTriggerDestroy,
Steps: []resource.TestStep{
{
Config: testAccAWSGlueTriggerConfig_Crawler(rName, "SUCCEEDED"),
Check: resource.ComposeTestCheckFunc(
testAccCheckAWSGlueTriggerExists(resourceName, &trigger),
resource.TestCheckResourceAttr(resourceName, "actions.#", "1"),
resource.TestCheckResourceAttr(resourceName, "actions.0.crawler_name", rName),
resource.TestCheckResourceAttr(resourceName, "predicate.#", "1"),
resource.TestCheckResourceAttr(resourceName, "predicate.0.conditions.#", "1"),
resource.TestCheckResourceAttr(resourceName, "predicate.0.conditions.0.crawler_name", fmt.Sprintf("%scrawl2", rName)),
resource.TestCheckResourceAttr(resourceName, "predicate.0.conditions.0.crawl_state", "SUCCEEDED"),
resource.TestCheckResourceAttr(resourceName, "type", "CONDITIONAL"),
),
},
{
Config: testAccAWSGlueTriggerConfig_Crawler(rName, "FAILED"),
Check: resource.ComposeTestCheckFunc(
testAccCheckAWSGlueTriggerExists(resourceName, &trigger),
resource.TestCheckResourceAttr(resourceName, "actions.#", "1"),
resource.TestCheckResourceAttr(resourceName, "actions.0.crawler_name", rName),
resource.TestCheckResourceAttr(resourceName, "predicate.#", "1"),
resource.TestCheckResourceAttr(resourceName, "predicate.0.conditions.#", "1"),
resource.TestCheckResourceAttr(resourceName, "predicate.0.conditions.0.crawler_name", fmt.Sprintf("%scrawl2", rName)),
resource.TestCheckResourceAttr(resourceName, "predicate.0.conditions.0.crawl_state", "FAILED"),
resource.TestCheckResourceAttr(resourceName, "type", "CONDITIONAL"),
),
},
{
Expand Down Expand Up @@ -239,6 +286,33 @@ func TestAccAWSGlueTrigger_Schedule(t *testing.T) {
})
}

func TestAccAWSGlueTrigger_WorkflowName(t *testing.T) {
var trigger glue.Trigger

rName := fmt.Sprintf("tf-acc-test-%s", acctest.RandString(5))
resourceName := "aws_glue_trigger.test"

resource.ParallelTest(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
Providers: testAccProviders,
CheckDestroy: testAccCheckAWSGlueTriggerDestroy,
Steps: []resource.TestStep{
{
Config: testAccAWSGlueTriggerConfig_WorkflowName(rName),
Check: resource.ComposeTestCheckFunc(
testAccCheckAWSGlueTriggerExists(resourceName, &trigger),
resource.TestCheckResourceAttr(resourceName, "workflow_name", rName),
),
},
{
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
},
},
})
}

func testAccCheckAWSGlueTriggerExists(resourceName string, trigger *glue.Trigger) resource.TestCheckFunc {
return func(s *terraform.State) error {
rs, ok := s.RootModule().Resources[resourceName]
Expand Down Expand Up @@ -383,6 +457,40 @@ resource "aws_glue_trigger" "test" {
`, testAccAWSGlueJobConfig_Required(rName), rName, rName, state)
}

func testAccAWSGlueTriggerConfig_Crawler(rName, state string) string {
return fmt.Sprintf(`
%s
resource "aws_glue_crawler" "test2" {
depends_on = ["aws_iam_role_policy_attachment.test-AWSGlueServiceRole"]
database_name = "${aws_glue_catalog_database.test.name}"
name = "%scrawl2"
role = "${aws_iam_role.test.name}"
s3_target {
path = "s3://test_bucket"
}
}
resource "aws_glue_trigger" "test_trigger" {
name = "%strigger"
type = "CONDITIONAL"
actions {
crawler_name = "${aws_glue_crawler.test.name}"
}
predicate {
conditions {
crawler_name = "${aws_glue_crawler.test2.name}"
crawl_state = "%s"
}
}
}
`, testAccGlueCrawlerConfig_S3Target(rName, "s3://test_bucket"), rName, rName, state)
}

func testAccAWSGlueTriggerConfig_Schedule(rName, schedule string) string {
return fmt.Sprintf(`
%s
Expand All @@ -398,3 +506,23 @@ resource "aws_glue_trigger" "test" {
}
`, testAccAWSGlueJobConfig_Required(rName), rName, schedule)
}

func testAccAWSGlueTriggerConfig_WorkflowName(rName string) string {
return fmt.Sprintf(`
%s
resource "aws_glue_workflow" test {
name = "%s"
}
resource "aws_glue_trigger" "test" {
name = "%s"
type = "ON_DEMAND"
workflow_name = "${aws_glue_workflow.test.name}"
actions {
job_name = "${aws_glue_job.test.name}"
}
}
`, testAccAWSGlueJobConfig_Required(rName), rName, rName)
}
Loading

0 comments on commit 5c9e5e0

Please sign in to comment.