Merge pull request #9000 from ministryofjustice/DBA-790

Dba 790

terraform/environments/delius-core/modules/components/dms/cloudwatch-alarms.tf

@@ -217,13 +217,17 @@ resource "aws_iam_role_policy_attachment" "lambda_put_metric_data_logging_attach
   policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
 }
 
+locals {
+  rendered_metric_template = templatefile("${path.module}/lambda/dms_replication_metric.py.tmpl",{oracle_db_instance_scheduling=var.oracle_db_instance_scheduling})
+}
+
 # Creates a ZIP file containing the contents of the lambda directory which
 # contains a Python script to calculate and put the custom metric
 data "archive_file" "lambda_dms_replication_metric_zip" {
   type        = "zip"
-  source_dir  = "${path.module}/lambda"
-  output_path = "${path.module}/lambda/dms_replication_metric.zip"
-  excludes    = ["dms_replication_metric.zip"]
+  source_content          = local.rendered_metric_template 
+  source_content_filename = "dms_replication_metric.py"
+  output_path             = "${path.module}/lambda/dms_replication_metric.zip"
 }
 
 # Define a Lambda Function using the python script in the ZIP file -

terraform/environments/delius-core/modules/components/dms/lambda/dms_replication_metric.py → terraform/environments/delius-core/modules/components/dms/lambda/dms_replication_metric.py.tmpl

@@ -2,15 +2,49 @@
 import json
 import logging
 import re
+from datetime import datetime
 
 logger = logging.getLogger()
 logger.setLevel(logging.INFO)
 
 def lambda_handler(event, context):
 
+    # The following guide describes Instance Scheduling - 
+    #    automatically stop non-production instances overnight
+    # https://user-guide.modernisation-platform.service.justice.gov.uk/concepts/environments/instance-scheduling.html#instance-scheduling-automatically-stop-non-production-instances-overnight
+    # 
+    # We should ignore DMS events sent when the primary database instance is not meant
+    # to be up.   That is under the following conditions:
+    # Before 6am or after 9pm, or all day weekends
+    # Except if the instance scheduling is set to "skip-scheduling" or "skip-auto-stop"
+    # as we do not automatically stop those instances.
+    # The instance scheduling for the environment is embedded into this function at
+    # creation time using the value of the delius primary database instance_scheduling tag;
+    # this avoids having to find and read the tag each time the function gets called as
+    # this instance_scheduling tag is unlikely to change very often if at all.
+    #
+    # If we ignore DMS events out of hours then that avoids changing the status of the
+    # CloudWatch alarm.  This is important because a DMS event is raised when replication
+    # fails overnight (due to the database instance being down), but *NO* DMS event is
+    # raised when the replication starts working again (due to the database instance
+    # being restarted).  Therefore we must avoid processing the overnight failure events.
+
     cloudwatch = boto3.client('cloudwatch')
     for record in event['Records']:
 
+        # Get the current local time and weekday (timezones are respected)
+        now = datetime.now()
+        current_hour    = now.hour
+        current_weekday = now.weekday()  # Monday is 0, Sunday is 6
+
+        # Skip the iteration if before 6am, after 9pm, or it is the weekend.
+        # These times are hard-coded into the stop/start functionality.  We do not
+        # currently support custom times.
+        # Do NOT skip if the instance_scheduling tag has the value of skip-scheduling 
+        # or skip-audit-stop is set as we must handle all DMS events in these environments (they are up 24/7)
+        if (current_hour < 6 or current_hour > 21 or current_weekday >= 5) and (not "${oracle_db_instance_scheduling}" in ["skip-scheduling","skip-auto-stop"]):
+            continue
+
         message = json.loads(record['Sns']['Message'])
         logger.info("SNS Message: %s",message)
 

terraform/environments/delius-core/modules/components/dms/variables.tf

@@ -69,4 +69,9 @@ variable "db_ec2_sg_id" {
 variable "env_name_to_dms_config_map" {
   description = "Map of delius-core environments to DMS configs"
   type        = any
+}
+
+variable "oracle_db_instance_scheduling" {
+  description = "instance_scheduling value.  See https://user-guide.modernisation-platform.service.justice.gov.uk/concepts/environments/instance-scheduling.html"
+  type        = string
 }

terraform/environments/delius-core/modules/components/oracle_db_instance/outputs.tf

@@ -1,3 +1,7 @@
 output "oracle_db_server_name" {
   value = module.instance.aws_instance.tags.server-name
+}
+
+output "oracle_db_instance_scheduling" {
+  value = try(module.instance.aws_instance.tags.instance-scheduling,"default")
 }

terraform/environments/delius-core/modules/delius_environment/dms.tf

@@ -10,6 +10,7 @@ module "dms" {
 
   database_application_passwords_secret_arn = module.oracle_db_shared.database_application_passwords_secret_arn
   oracle_db_server_names                    = local.oracle_db_server_names
+  oracle_db_instance_scheduling             = module.oracle_db_primary[0].oracle_db_instance_scheduling
   db_ec2_sg_id                              = module.oracle_db_shared.db_ec2_sg_id
   env_name_to_dms_config_map                = var.env_name_to_dms_config_map