diff --git a/terraform/core/13-mssql-ingestion.tf b/terraform/core/13-mssql-ingestion.tf index 0d93e1640..d6d187f6d 100644 --- a/terraform/core/13-mssql-ingestion.tf +++ b/terraform/core/13-mssql-ingestion.tf @@ -62,8 +62,9 @@ resource "aws_glue_trigger" "filter_ingestion_tables" { } module "ingest_academy_revenues_and_benefits_housing_needs_to_landing_zone" { - for_each = local.table_filter_expressions - tags = module.tags.values + trigger_enabled = false + for_each = local.table_filter_expressions + tags = module.tags.values source = "../modules/aws-glue-job" is_live_environment = local.is_live_environment @@ -201,6 +202,7 @@ module "copy_academy_revenues_to_raw_zone" { locals { academy_state_machine_count = local.is_live_environment && !local.is_production_environment ? 1 : 0 number_of_glue_workers = 2 + academy_table_filters = ["^lbhaliverbviews_core_hbrent[s].*", "^lbhaliverbviews_core_hbc.*", "^lbhaliverbviews_core_hbuc.*", "^lbhaliverbviews_core_hbrentclaim", "^lbhaliverbviews_core_hbrenttrans", "^lbhaliverbviews_core_hbrent[^tsc].*", "^lbhaliverbviews_core_hbmember", "^lbhaliverbviews_core_hbincome", "^lbhaliverbviews_core_hb[abdefghjklnopsw]", "^lbhaliverbviews_core_ct[dt].*", "^lbhaliverbviews_current_ctax.*", "^lbhaliverbviews_current_[hbn].*", "^lbhaliverbviews_core_ct[abcefghijklmnopqrsvw].*", "^lbhaliverbviews_core_recov_event", "(^lbhaliverbviews_core_cr.*|^lbhaliverbviews_core_[ins].*|^lbhaliverbviews_xdbvw.*|^lbhaliverbviews_current_im.*)"] } module "academy_glue_job" { @@ -210,20 +212,21 @@ module "academy_glue_job" { is_live_environment = local.is_live_environment is_production_environment = local.is_production_environment - job_name = "${local.short_identifier_prefix}Academy Revs & Bens Housing Needs Database Ingestion" - script_s3_object_key = aws_s3_object.ingest_database_tables_via_jdbc_connection.key - environment = var.environment - pydeequ_zip_key = aws_s3_object.pydeequ.key - helper_module_key = aws_s3_object.helpers.key - jdbc_connections = [module.academy_mssql_database_ingestion[0].jdbc_connection_name] - glue_role_arn = aws_iam_role.glue_role.arn - glue_temp_bucket_id = module.glue_temp_storage.bucket_id - glue_scripts_bucket_id = module.glue_scripts.bucket_id - spark_ui_output_storage_id = module.spark_ui_output_storage.bucket_id - glue_job_timeout = 420 - glue_version = "4.0" - glue_job_worker_type = "G.1X" - number_of_workers_for_glue_job = local.number_of_glue_workers + job_name = "${local.short_identifier_prefix}Academy Revs & Bens Housing Needs Database Ingestion" + script_s3_object_key = aws_s3_object.ingest_database_tables_via_jdbc_connection.key + environment = var.environment + pydeequ_zip_key = aws_s3_object.pydeequ.key + helper_module_key = aws_s3_object.helpers.key + jdbc_connections = [module.academy_mssql_database_ingestion[0].jdbc_connection_name] + glue_role_arn = aws_iam_role.glue_role.arn + glue_temp_bucket_id = module.glue_temp_storage.bucket_id + glue_scripts_bucket_id = module.glue_scripts.bucket_id + spark_ui_output_storage_id = module.spark_ui_output_storage.bucket_id + glue_job_timeout = 420 + glue_version = "4.0" + glue_job_worker_type = "G.1X" + number_of_workers_for_glue_job = local.number_of_glue_workers + max_concurrent_runs_of_glue_job = length(local.academy_table_filters) job_parameters = { "--source_data_database" = module.academy_mssql_database_ingestion[0].ingestion_database_name "--s3_ingestion_bucket_target" = "s3://${module.landing_zone.bucket_id}/academy/" @@ -374,3 +377,65 @@ data "aws_iam_policy_document" "academy_step_functions_policy" { resources = ["*"] } } + +resource "aws_cloudwatch_event_rule" "academy_state_machine_trigger" { + count = local.academy_state_machine_count + name = "${local.short_identifier_prefix}academy-state-machine-trigger" + tags = module.tags.values + description = "Trigger the Academy State Machine every weekday at 1am" + schedule_expression = "cron(0 1 ? * MON-FRI *)" + is_enabled = true + role_arn = aws_iam_role.academy_cloudwatch_execution_role[0].arn +} + +resource "aws_cloudwatch_event_target" "academy_state_machine_trigger" { + count = local.academy_state_machine_count + rule = aws_cloudwatch_event_rule.academy_state_machine_trigger[0].name + arn = module.academy_state_machine[0].arn + input = <