DPR2-1492: Reduce polling frequency when checking processed files (#8889

)

terraform/environments/digital-prison-reporting/modules/domains/maintenance-pipeline/pipeline.tf

@@ -30,7 +30,8 @@ module "maintenance_pipeline" {
           "Parameters" : {
             "JobName" : var.glue_unprocessed_raw_files_check_job,
             "Arguments" : {
-              "--dpr.orchestration.wait.interval.seconds" : "60",
+              "--dpr.orchestration.wait.interval.seconds" : tostring(var.processed_files_check_wait_interval_seconds),
+              "--dpr.orchestration.max.attempts" : tostring(var.processed_files_check_max_attempts),
               "--dpr.datastorage.retry.maxAttempts" : tostring(var.glue_s3_max_attempts),
               "--dpr.datastorage.retry.minWaitMillis" : tostring(var.glue_s3_retry_min_wait_millis),
               "--dpr.datastorage.retry.maxWaitMillis" : tostring(var.glue_s3_retry_max_wait_millis)

terraform/environments/digital-prison-reporting/modules/domains/maintenance-pipeline/variables.tf

@@ -157,6 +157,16 @@ variable "retention_curated_num_workers" {
   }
 }
 
+variable "processed_files_check_wait_interval_seconds" {
+  description = "Amount of seconds between checks to s3 if all files have been processed"
+  type        = number
+}
+
+variable "processed_files_check_max_attempts" {
+  description = "Maximum number of attempts to check if all files have been processed"
+  type        = number
+}
+
 variable "glue_s3_max_attempts" {
   description = "The maximum number of attempts when making requests to S3"
   type        = number

terraform/environments/digital-prison-reporting/modules/domains/reload-pipeline/pipeline.tf

@@ -53,7 +53,8 @@ module "reload_pipeline" {
           "Parameters" : {
             "JobName" : var.glue_unprocessed_raw_files_check_job,
             "Arguments" : {
-              "--dpr.orchestration.wait.interval.seconds" : "60"
+              "--dpr.orchestration.wait.interval.seconds" : tostring(var.processed_files_check_wait_interval_seconds),
+              "--dpr.orchestration.max.attempts" : tostring(var.processed_files_check_max_attempts)
             }
           },
           "Next" : "Stop Glue Streaming Job"

terraform/environments/digital-prison-reporting/modules/domains/reload-pipeline/variables.tf

@@ -238,6 +238,16 @@ variable "retention_curated_num_workers" {
   }
 }
 
+variable "processed_files_check_wait_interval_seconds" {
+  description = "Amount of seconds between checks to s3 if all files have been processed"
+  type        = number
+}
+
+variable "processed_files_check_max_attempts" {
+  description = "Maximum number of attempts to check if all files have been processed"
+  type        = number
+}
+
 variable "glue_s3_max_attempts" {
   description = "The maximum number of attempts when making requests to S3"
   type        = number

terraform/environments/digital-prison-reporting/modules/domains/replay-pipeline/pipeline.tf

@@ -52,7 +52,8 @@ module "replay_pipeline" {
           "Parameters" : {
             "JobName" : var.glue_unprocessed_raw_files_check_job,
             "Arguments" : {
-              "--dpr.orchestration.wait.interval.seconds" : "60"
+              "--dpr.orchestration.wait.interval.seconds" : tostring(var.processed_files_check_wait_interval_seconds),
+              "--dpr.orchestration.max.attempts" : tostring(var.processed_files_check_max_attempts)
             }
           },
           "Next" : "Stop Glue Streaming Job"
@@ -250,7 +251,8 @@ module "replay_pipeline" {
           "Parameters" : {
             "JobName" : var.glue_unprocessed_raw_files_check_job,
             "Arguments" : {
-              "--dpr.orchestration.wait.interval.seconds" : "60"
+              "--dpr.orchestration.wait.interval.seconds" : tostring(var.processed_files_check_wait_interval_seconds),
+              "--dpr.orchestration.max.attempts" : tostring(var.processed_files_check_max_attempts)
             }
           },
           "Next" : "Empty Raw Data"

terraform/environments/digital-prison-reporting/modules/domains/replay-pipeline/variables.tf

@@ -235,6 +235,16 @@ variable "retention_curated_num_workers" {
   }
 }
 
+variable "processed_files_check_wait_interval_seconds" {
+  description = "Amount of seconds between checks to s3 if all files have been processed"
+  type        = number
+}
+
+variable "processed_files_check_max_attempts" {
+  description = "Maximum number of attempts to check if all files have been processed"
+  type        = number
+}
+
 variable "glue_s3_max_attempts" {
   description = "The maximum number of attempts when making requests to S3"
   type        = number

terraform/environments/digital-prison-reporting/modules/domains/stop-cdc-pipeline/pipeline.tf

@@ -29,8 +29,8 @@ module "cdc_stop_pipeline" {
           "Parameters" : {
             "JobName" : var.glue_unprocessed_raw_files_check_job,
             "Arguments" : {
-              "--dpr.orchestration.wait.interval.seconds" : "60"
-              "--dpr.orchestration.max.attempts" : "120"
+              "--dpr.orchestration.wait.interval.seconds" : tostring(var.processed_files_check_wait_interval_seconds),
+              "--dpr.orchestration.max.attempts" : tostring(var.processed_files_check_max_attempts)
             }
           },
           "Next" : "Stop Glue Streaming Job"

terraform/environments/digital-prison-reporting/modules/domains/stop-cdc-pipeline/variables.tf

@@ -39,6 +39,16 @@ variable "glue_unprocessed_raw_files_check_job" {
   type        = string
 }
 
+variable "processed_files_check_wait_interval_seconds" {
+  description = "Amount of seconds between checks to s3 if all files have been processed"
+  type        = number
+}
+
+variable "processed_files_check_max_attempts" {
+  description = "Maximum number of attempts to check if all files have been processed"
+  type        = number
+}
+
 variable "tags" {
   type        = map(string)
   default     = {}