Update 24-aws-glue-spatial.tf

4 workers for file loader job
LBHackney-IT · Oct 27, 2023 · d030873 · d030873
1 parent 133ea66
commit d030873
Showing 1 changed file with 8 additions and 7 deletions.
diff --git a/terraform/etl/24-aws-glue-spatial.tf b/terraform/etl/24-aws-glue-spatial.tf
@@ -157,13 +157,14 @@ module "addressbasepremium_load_files" {
   is_live_environment       = local.is_live_environment
   is_production_environment = local.is_production_environment
 
-  department                 = module.department_unrestricted_data_source
-  job_name                   = "${local.short_identifier_prefix}addressbasepremium_load_files"
-  glue_version               = "4.0"
-  glue_job_worker_type       = "G.1X"
-  helper_module_key          = data.aws_s3_object.helpers.key
-  pydeequ_zip_key            = data.aws_s3_object.pydeequ.key
-  spark_ui_output_storage_id = module.spark_ui_output_storage_data_source.bucket_id
+  department                     = module.department_unrestricted_data_source
+  job_name                       = "${local.short_identifier_prefix}addressbasepremium_load_files"
+  glue_version                   = "4.0"
+  glue_job_worker_type           = "G.1X"
+  number_of_workers_for_glue_job = 4
+  helper_module_key              = data.aws_s3_object.helpers.key
+  pydeequ_zip_key                = data.aws_s3_object.pydeequ.key
+  spark_ui_output_storage_id     = module.spark_ui_output_storage_data_source.bucket_id
   job_parameters = {
     "--job-bookmark-option"  = "job-bookmark-disable"    
     "--raw_bucket"           = module.raw_zone_data_source.bucket_id