Skip to content

Commit

Permalink
add temporary triggers on crawlers
Browse files Browse the repository at this point in the history
  • Loading branch information
Tian-2017 committed Jan 8, 2025
1 parent 44a8ce5 commit 1201403
Showing 1 changed file with 37 additions and 1 deletion.
38 changes: 37 additions & 1 deletion terraform/etl/60-airflow-etl-used-crawlers.tf
Original file line number Diff line number Diff line change
Expand Up @@ -152,4 +152,40 @@ resource "aws_glue_crawler" "data_and_insight_google_sheet_ingestion_raw_zone" {
Tables = { AddOrUpdateBehavior = "MergeNewColumns" }
}
})
}
}

# Below crawlers triggers are temporary and will be removed after enabling the airflow google sheet ingestion dag
resource "aws_glue_trigger" "parking_google_sheet_ingestion_raw_zone_trigger" {
name = "${local.short_identifier_prefix}${module.department_parking_data_source.identifier}-google-sheet-ingestion-raw-zone-trigger"
type = "SCHEDULED"
schedule = "cron(0 7 ? * * *)"
start_on_creation = true

actions {
crawler_name = aws_glue_crawler.parking_google_sheet_ingestion_raw_zone.name
}
}


resource "aws_glue_trigger" "housing_google_sheet_ingestion_raw_zone_trigger" {
name = "${local.short_identifier_prefix}${module.department_housing_data_source.identifier}-google-sheet-ingestion-raw-zone-trigger"
type = "SCHEDULED"
schedule = "cron(0 7 ? * * *)"
start_on_creation = true

actions {
crawler_name = aws_glue_crawler.housing_google_sheet_ingestion_raw_zone.name
}
}


resource "aws_glue_trigger" "data_and_insight_google_sheet_ingestion_raw_zone_trigger" {
name = "${local.short_identifier_prefix}${module.department_data_and_insight_data_source.identifier}-google-sheet-ingestion-raw-zone-trigger"
type = "SCHEDULED"
schedule = "cron(0 7 ? * * *)"
start_on_creation = true

actions {
crawler_name = aws_glue_crawler.data_and_insight_google_sheet_ingestion_raw_zone.name
}
}

0 comments on commit 1201403

Please sign in to comment.