Dpp 558 liberator ingestion

terraform/core/10-aws-s3-buckets.tf

@@ -424,3 +424,25 @@ resource "aws_s3_bucket_versioning" "ssl_connection_resources" {
     status = "Enabled"
   }
 }
+
+module "rds_export_storage" {
+  source = "../modules/s3-bucket"
+
+  tags              = module.tags.values
+  project           = var.project
+  environment       = var.environment
+  identifier_prefix = local.identifier_prefix
+  bucket_name       = "RDS Export Storage"
+  bucket_identifier = "rds-export-storage"
+}
+
+resource "aws_s3_bucket_server_side_encryption_configuration" "rds_export_storage_encryption" {
+  bucket = module.rds_export_storage.bucket_id
+
+  rule {
+    apply_server_side_encryption_by_default {
+      sse_algorithm = "aws:kms"
+    }
+    bucket_key_enabled = true
+  }
+}

terraform/core/29-db-snapshot-to-s3-sandbox.tf

@@ -2,13 +2,13 @@
 
 # 1. Deploy this first to get the database and bastion host in place
 module "db_snapshot_to_s3_sandbox_resources" {
-  count                   = 0
-  source                  = "../modules/db-snapshot-to-s3-sandbox-resources"
-  tags                    = module.tags.values
-  identifier_prefix       = local.identifier_prefix
-  aws_sandbox_subnet_ids  = var.aws_sandbox_subnet_ids
-  aws_sandbox_account_id  = var.aws_sandbox_account_id
-  aws_sandbox_vpc_id      = var.aws_sandbox_vpc_id
+  count                  = 0
+  source                 = "../modules/db-snapshot-to-s3-sandbox-resources"
+  tags                   = module.tags.values
+  identifier_prefix      = local.identifier_prefix
+  aws_sandbox_subnet_ids = var.aws_sandbox_subnet_ids
+  aws_sandbox_account_id = var.aws_sandbox_account_id
+  aws_sandbox_vpc_id     = var.aws_sandbox_vpc_id
 
   providers = {
     aws                     = aws
@@ -20,7 +20,7 @@ module "db_snapshot_to_s3_sandbox_resources" {
 
 # 3. lambda_artefact_storage_for_sandbox_account and db_snapshot_to_s3_sandbox can be deployed at the same time
 module "lambda_artefact_storage_for_sandbox_account" {
-  count = 0
+  count             = 0
   source            = "../modules/s3-bucket"
   tags              = module.tags.values
   project           = var.project
@@ -34,33 +34,35 @@ module "lambda_artefact_storage_for_sandbox_account" {
   }
 }
 
-module "db_snapshot_to_s3_sandbox" {
-  count = 0
-  source                         = "../modules/db-snapshot-to-s3"
-  tags                           = module.tags.values
-  project                        = var.project
-  environment                    = var.environment
-  identifier_prefix              = local.identifier_prefix
-  lambda_artefact_storage_bucket = module.lambda_artefact_storage_for_sandbox_account[0].bucket_id
-  zone_kms_key_arn               = module.raw_zone.kms_key_arn
-  zone_bucket_arn                = module.raw_zone.bucket_arn
-  zone_bucket_id                 = module.raw_zone.bucket_id
-  service_area                   = "unrestricted"
-  rds_instance_ids               = var.rds_instance_ids
-  aws_account_suffix             = "-sandbox"
-
-  providers = {
-    aws = aws.aws_sandbox_account
-  }
-}
+#module "db_snapshot_to_s3_sandbox" {
+#  count                          = 0
+#  source                         = "../modules/db-snapshot-to-s3"
+#  tags                           = module.tags.values
+#  project                        = var.project
+#  environment                    = var.environment
+#  identifier_prefix              = local.identifier_prefix
+#  lambda_artefact_storage_bucket = module.lambda_artefact_storage_for_sandbox_account[0].bucket_id
+#  zone_kms_key_arn               = module.raw_zone.kms_key_arn
+#  zone_bucket_arn                = module.raw_zone.bucket_arn
+#  zone_bucket_id                 = module.raw_zone.bucket_id
+#  rds_export_storage_bucket_arn  = module.rds_export_storage.bucket_arn
+#  rds_export_storage_kms_key_arn = module.rds_export_storage.kms_key_arn
+#  service_area                   = "unrestricted"
+#  rds_instance_ids               = var.rds_instance_ids
+#  aws_account_suffix             = "-sandbox"
+#
+#  providers = {
+#    aws = aws.aws_sandbox_account
+#  }
+#}
 
 #4. Update the raw zone bucket on DP dev account in your workspace with the following bucket and bucket key statements
 #   Use these as inputs for bucket_policy_statements and bucket_key_policy_statements in the raw zone bucket module
 
 # sandbox_s3_to_s3_copier_write_access_to_raw_zone_statement = {
 #     sid = "AllowSandboxS3toS3CopierWriteAccessToRawZoneUnrestrictedLocation"
 #     effect = "Allow"
-   
+
 #     actions = [
 #       "s3:ListBucket",
 #       "s3:PutObject",
@@ -95,7 +97,7 @@ module "db_snapshot_to_s3_sandbox" {
 #         "arn:aws:iam::${var.aws_sandbox_account_id}:role/${local.identifier_prefix}-s3-to-s3-copier-lambda"
 #       ]
 #     }
-  
+
 #   }
 
 #5. Uncomment the statement in the sandbox database key policy to allow the rds snapshot to s3 lambda role access to the key. This must be done after all other resources have been deployed.

terraform/core/29-db-snapshot-to-s3.tf

@@ -23,6 +23,10 @@ module "db_snapshot_to_s3" {
   zone_kms_key_arn               = module.raw_zone.kms_key_arn
   zone_bucket_arn                = module.raw_zone.bucket_arn
   zone_bucket_id                 = module.raw_zone.bucket_id
+  rds_export_storage_bucket_arn  = module.rds_export_storage.bucket_arn
+  rds_export_storage_bucket_id   = module.rds_export_storage.bucket_id
+  rds_export_storage_kms_key_arn = module.rds_export_storage.kms_key_arn
+  rds_export_storage_kms_key_id  = module.rds_export_storage.kms_key_id
   service_area                   = "unrestricted"
   rds_instance_ids               = var.rds_instance_ids
 

terraform/core/36-liberator-import.tf

@@ -37,6 +37,10 @@ module "liberator_db_snapshot_to_s3" {
   zone_kms_key_arn               = module.landing_zone.kms_key_arn
   zone_bucket_arn                = module.landing_zone.bucket_arn
   zone_bucket_id                 = module.landing_zone.bucket_id
+  rds_export_storage_bucket_arn  = module.rds_export_storage.bucket_arn
+  rds_export_storage_bucket_id   = module.rds_export_storage.bucket_id
+  rds_export_storage_kms_key_arn = module.rds_export_storage.kms_key_arn
+  rds_export_storage_kms_key_id  = module.rds_export_storage.kms_key_id
   service_area                   = "parking"
   rds_instance_ids               = [for item in module.liberator_dump_to_rds_snapshot : item.rds_instance_id]
   workflow_name                  = aws_glue_workflow.parking_liberator_data.name
@@ -79,16 +83,19 @@ data "aws_iam_policy_document" "lambda_assume_role" {
 ### New modules for liberator ingestion
 
 module "liberator_rds_snapshot_to_s3" {
-  count                          = 0
+  count                          = 1
   source                         = "../modules/rds-snapshot-to-s3"
   tags                           = module.tags.values
   identifier_prefix              = local.identifier_prefix
   project                        = var.project
   environment                    = var.environment
   lambda_artefact_storage_bucket = module.lambda_artefact_storage.bucket_id
   zone_kms_key_arn               = module.landing_zone.kms_key_arn
-  zone_bucket_arn                = module.landing_zone.bucket_arn
+  source_bucket_arn              = module.landing_zone.bucket_arn
   zone_bucket_id                 = module.landing_zone.bucket_id
+  target_bucket_arn              = module.raw_zone.bucket_arn
+  source_prefix                  = "parking/liberator/"
+  target_prefix                  = "parking/liberator/"
   service_area                   = "parking"
   rds_instance_ids               = [for item in module.liberator_dump_to_rds_snapshot : item.rds_instance_id]
   rds_instance_arns              = [for item in module.liberator_dump_to_rds_snapshot : item.rds_instance_arn]

terraform/modules/db-snapshot-to-s3/01-inputs-required.tf

@@ -42,3 +42,19 @@ variable "service_area" {
 variable "rds_instance_ids" {
   type = list(string)
 }
+
+variable "rds_export_storage_bucket_arn" {
+  type = string
+}
+
+variable "rds_export_storage_kms_key_arn" {
+  type = string
+}
+
+variable "rds_export_storage_kms_key_id" {
+  type = string
+}
+
+variable "rds_export_storage_bucket_id" {
+  type = string
+}

terraform/modules/db-snapshot-to-s3/20-rds-to-s3-lambda.tf

@@ -159,8 +159,8 @@ resource "aws_lambda_function" "rds_snapshot_to_s3_lambda" {
   environment {
     variables = {
       IAM_ROLE_ARN     = aws_iam_role.rds_snapshot_export_service.arn,
-      KMS_KEY_ID       = module.rds_export_storage.kms_key_id,
-      S3_BUCKET_NAME   = module.rds_export_storage.bucket_id,
+      KMS_KEY_ID       = var.rds_export_storage_kms_key_id,
+      S3_BUCKET_NAME   = var.rds_export_storage_bucket_id,
       COPIER_QUEUE_ARN = aws_sqs_queue.s3_to_s3_copier.arn
     }
   }

terraform/modules/db-snapshot-to-s3/25-rds-to-s3-queue.tf

@@ -128,7 +128,7 @@ resource "aws_sns_topic_subscription" "subscribe_sqs_to_sns_topic" {
 
 resource "aws_lambda_event_source_mapping" "event_source_mapping" {
   event_source_arn = aws_sqs_queue.rds_snapshot_to_s3.arn
-  enabled          = true
+  enabled          = false
   function_name    = aws_lambda_function.rds_snapshot_to_s3_lambda.arn
   batch_size       = 1
 }

terraform/modules/db-snapshot-to-s3/30-rds-snapshot-export-service.tf

@@ -41,10 +41,8 @@ data "aws_iam_policy_document" "rds_snapshot_export_service" {
       "s3:DeleteObject*"
     ]
     resources = [
-      module.rds_export_storage.bucket_arn,
-      "${module.rds_export_storage.bucket_arn}/*",
-      module.rds_export_storage.bucket_arn,
-      "${module.rds_export_storage.bucket_arn}/*",
+      var.rds_export_storage_bucket_arn,
+      "${var.rds_export_storage_bucket_arn}/*"
     ]
   }
 
@@ -54,7 +52,7 @@ data "aws_iam_policy_document" "rds_snapshot_export_service" {
     ]
     effect = "Allow"
     resources = [
-      module.rds_export_storage.kms_key_arn
+      var.rds_export_storage_kms_key_arn
     ]
   }
 }

terraform/modules/db-snapshot-to-s3/40-s3-to-s3-copier-lambda.tf

@@ -53,8 +53,8 @@ data "aws_iam_policy_document" "s3_to_s3_copier_lambda" {
     ]
     effect = "Allow"
     resources = [
-      module.rds_export_storage.kms_key_arn,
-      "${module.rds_export_storage.bucket_arn}/*",
+      var.rds_export_storage_bucket_arn,
+      "${var.rds_export_storage_bucket_arn}/*",
       var.zone_kms_key_arn,
       var.zone_bucket_arn,
       "${var.zone_bucket_arn}/*",
@@ -73,8 +73,8 @@ data "aws_iam_policy_document" "s3_to_s3_copier_lambda" {
     resources = [
       var.zone_bucket_arn,
       "${var.zone_bucket_arn}/*",
-      module.rds_export_storage.bucket_arn,
-      "${module.rds_export_storage.bucket_arn}/*"
+      var.rds_export_storage_bucket_arn,
+      "${var.rds_export_storage_bucket_arn}/*"
     ]
   }
 

terraform/modules/db-snapshot-to-s3/45-s3-to-s3-copier-queue.tf

@@ -13,7 +13,7 @@ resource "aws_sqs_queue" "s3_to_s3_copier" {
   visibility_timeout_seconds = local.lambda_timeout * 6
 
   name              = lower("${var.identifier_prefix}-s3-to-s3-copier")
-  kms_master_key_id = aws_kms_key.s3_to_s3_copier_kms_key.key_id
+  kms_master_key_id = var.rds_export_storage_kms_key_id
 }
 
 resource "aws_kms_key" "s3_to_s3_copier_kms_key" {

terraform/modules/rds-snapshot-to-s3/01-inputs-required.tf

@@ -26,9 +26,6 @@ variable "zone_kms_key_arn" {
   type = string
 }
 
-variable "zone_bucket_arn" {
-  type = string
-}
 
 variable "zone_bucket_id" {
   type = string
@@ -46,3 +43,11 @@ variable "rds_instance_ids" {
 variable "rds_instance_arns" {
   type = list(string)
 }
+
+variable "source_bucket_arn" {
+  type = string
+}
+
+variable "target_bucket_arn" {
+  type = string
+}

terraform/modules/rds-snapshot-to-s3/02-inputs-optional.tf

@@ -27,3 +27,15 @@ variable "aws_account_suffix" {
   type        = string
   default     = ""
 }
+
+variable "source_prefix" {
+  description = "Prefix to be used for the source bucket location"
+  type        = string
+  default     = ""
+}
+
+variable "target_prefix" {
+  description = "Prefix to be used for the target bucket location"
+  type        = string
+  default     = ""
+}

terraform/modules/rds-snapshot-to-s3/99-outputs.tf

@@ -1,4 +1,4 @@
-output "cloudwatch_event_rule_names" {
-  description = "The names of the CloudWatch Event Rules"
-  value       = [for rule in aws_cloudwatch_event_rule.rds_event_rule : rule.name]
-}
+output "rds_snapshot_s3_to_s3_copier_lambda_role_arn" {
+  description = "ARN for the s3_to_s3_copier_lambda_role"
+  value       = aws_iam_role.rds_snapshot_s3_to_s3_copier_lambda_role.arn
+}

terraform/modules/rds-snapshot-to-s3/eventbridge.tf

@@ -5,29 +5,54 @@ locals {
   }]
 }
 
-resource "aws_cloudwatch_event_rule" "rds_event_rule" {
+resource "aws_cloudwatch_event_rule" "rds_snapshot_created_event_rule" {
   for_each = { for instance in local.rds_instances : instance.id => instance }
 
-  name        = "rds-event-rule-${each.value.id}"
-  description = "Capture RDS Event 0161 for ${each.value.id}"
+  name        = "rds-event-rule-${each.value.id}-snapshot-created"
+  description = "Capture RDS Event 0042 (Snapshot Created) for ${each.value.id}"
 
   event_pattern = jsonencode({
-    source      = ["aws.rds"],
-    detail-type = ["RDS DB Instance Event"],
-    resources   = [each.value.arn],
+    source = ["aws.rds"],
     detail = {
-      EventCategories = ["snapshot"],
-      SourceType      = ["db-instance"],
-      Message         = ["RDS-EVENT-0161"]
+      SourceArn = [{
+        "prefix" : "arn:aws:rds:eu-west-2:120038763019:snapshot:sql-to-parquet"
+      }],
+      EventID = ["RDS-EVENT-0042"]
     }
   })
 
   tags = var.tags
 }
 
-resource "aws_cloudwatch_event_target" "rds_event_target" {
+resource "aws_cloudwatch_event_target" "rds_snapshot_created_event_target" {
   for_each = { for instance in local.rds_instances : instance.id => instance }
 
-  rule = aws_cloudwatch_event_rule.rds_event_rule[each.key].name
-  arn  = module.rds-to-s3-copier.lambda_function_arn
+  rule = aws_cloudwatch_event_rule.rds_snapshot_created_event_rule[each.key].name
+  arn  = module.trigger_rds_snapshot_export.lambda_function_arn
+}
+
+resource "aws_cloudwatch_event_rule" "rds_snapshot_exported_event_rule" {
+  for_each = { for instance in local.rds_instances : instance.id => instance }
+
+  name        = "rds-event-rule-${each.value.id}-snapshot-exported"
+  description = "Capture RDS Event 0161 (Snapshot Exported) for ${each.value.id}"
+
+  event_pattern = jsonencode({
+    source = ["aws.rds"],
+    detail = {
+      SourceArn = [{
+        "prefix" : "arn:aws:rds:eu-west-2:120038763019:snapshot:sql-to-parquet"
+      }],
+      EventID = ["RDS-EVENT-0161"]
+    }
+  })
+
+  tags = var.tags
+}
+
+resource "aws_cloudwatch_event_target" "rds_export_s3_to_s3_event_target" {
+  for_each = { for instance in local.rds_instances : instance.id => instance }
+
+  rule = aws_cloudwatch_event_rule.rds_snapshot_exported_event_rule[each.key].name
+  arn  = module.rds_snapshot_s3_to_s3_copier.lambda_function_arn
 }