From 6f0a976a78bd94fe33e9e5b40488f387174df208 Mon Sep 17 00:00:00 2001 From: timburke-hackit Date: Thu, 12 Oct 2023 09:07:26 +0100 Subject: [PATCH 01/17] reaname folder --- .../main.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename lambdas/{export-rds-snapshot-to-s3 => export_rds_snapshot_to_s3}/main.py (100%) diff --git a/lambdas/export-rds-snapshot-to-s3/main.py b/lambdas/export_rds_snapshot_to_s3/main.py similarity index 100% rename from lambdas/export-rds-snapshot-to-s3/main.py rename to lambdas/export_rds_snapshot_to_s3/main.py From 2b07696ca0816c857f9ee80135e27c951dd6ccdb Mon Sep 17 00:00:00 2001 From: timburke-hackit Date: Thu, 12 Oct 2023 09:25:59 +0100 Subject: [PATCH 02/17] add lambdas --- .../modules/rds-snapshot-to-s3/lambda.tf | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/terraform/modules/rds-snapshot-to-s3/lambda.tf b/terraform/modules/rds-snapshot-to-s3/lambda.tf index b11421373..ec99d52e0 100644 --- a/terraform/modules/rds-snapshot-to-s3/lambda.tf +++ b/terraform/modules/rds-snapshot-to-s3/lambda.tf @@ -1,12 +1,25 @@ -module "rds-to-s3-copier" { +module "trigger_rds_snapshot_export" { source = "../aws-lambda" lambda_name = "rds-to-s3-copier" - runtime = "python3.8" + runtime = "python3.10" handler = "lambda_function.lambda_handler" lambda_artefact_storage_bucket = var.lambda_artefact_storage_bucket - lambda_source_dir = "../../lambdas/s3-to-s3-export-copier-python" - lambda_output_path = "../../lambdas/rds-to-s3-copier.zip" - s3_key = "rds-to-s3-copier.zip" + lambda_source_dir = "../../lambdas/export_rds_snapshot_to_s3" + lambda_output_path = "../../lambdas/export-rds-snapshot-to-s3.zip" + s3_key = "export-rds-snapshot-to-s3.zip" identifier_prefix = var.identifier_prefix tags = var.tags } + +module "rds_snapshot_s3_to_s3_copier" { + source = "../aws-lambda" + lambda_name = "rds-export-s3-to-s3-copier" + runtime = "python3.10" + handler = "lambda_function.lambda_handler" + lambda_artefact_storage_bucket = var.lambda_artefact_storage_bucket + lambda_source_dir = "../../lambdas/rds_export_s3_to_s3_copier" + lambda_output_path = "../../lambdas/rds-export-s3-to-s3-copier.zip" + s3_key = "rds-export-s3-to-s3-copier.zip" + identifier_prefix = var.identifier_prefix + tags = var.tags +} \ No newline at end of file From 8b02b21b3041b012a2db585bd2b8197f7755f031 Mon Sep 17 00:00:00 2001 From: timburke-hackit Date: Thu, 12 Oct 2023 09:31:46 +0100 Subject: [PATCH 03/17] rename lambda --- terraform/modules/rds-snapshot-to-s3/lambda.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/modules/rds-snapshot-to-s3/lambda.tf b/terraform/modules/rds-snapshot-to-s3/lambda.tf index ec99d52e0..14f20ae54 100644 --- a/terraform/modules/rds-snapshot-to-s3/lambda.tf +++ b/terraform/modules/rds-snapshot-to-s3/lambda.tf @@ -1,6 +1,6 @@ module "trigger_rds_snapshot_export" { source = "../aws-lambda" - lambda_name = "rds-to-s3-copier" + lambda_name = "export-rds-snapshot-to-s3" runtime = "python3.10" handler = "lambda_function.lambda_handler" lambda_artefact_storage_bucket = var.lambda_artefact_storage_bucket From 11cad8b279cda6979081c3e0eafd170384663f0e Mon Sep 17 00:00:00 2001 From: timburke-hackit Date: Thu, 12 Oct 2023 11:14:02 +0100 Subject: [PATCH 04/17] remove event rule name output --- terraform/modules/rds-snapshot-to-s3/99-outputs.tf | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/terraform/modules/rds-snapshot-to-s3/99-outputs.tf b/terraform/modules/rds-snapshot-to-s3/99-outputs.tf index 03946c479..8b1378917 100644 --- a/terraform/modules/rds-snapshot-to-s3/99-outputs.tf +++ b/terraform/modules/rds-snapshot-to-s3/99-outputs.tf @@ -1,4 +1 @@ -output "cloudwatch_event_rule_names" { - description = "The names of the CloudWatch Event Rules" - value = [for rule in aws_cloudwatch_event_rule.rds_event_rule : rule.name] -} \ No newline at end of file + From 43a80da5148ad8b33ced9ee69ecbdaed07cb6ea2 Mon Sep 17 00:00:00 2001 From: timburke-hackit Date: Thu, 12 Oct 2023 11:14:31 +0100 Subject: [PATCH 05/17] eventbrige rules for triggering lambdas --- .../modules/rds-snapshot-to-s3/eventbridge.tf | 49 ++++++++++++++----- 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/terraform/modules/rds-snapshot-to-s3/eventbridge.tf b/terraform/modules/rds-snapshot-to-s3/eventbridge.tf index c45bae813..c9e1f6146 100644 --- a/terraform/modules/rds-snapshot-to-s3/eventbridge.tf +++ b/terraform/modules/rds-snapshot-to-s3/eventbridge.tf @@ -5,29 +5,54 @@ locals { }] } -resource "aws_cloudwatch_event_rule" "rds_event_rule" { +resource "aws_cloudwatch_event_rule" "rds_snapshot_created_event_rule" { for_each = { for instance in local.rds_instances : instance.id => instance } - name = "rds-event-rule-${each.value.id}" - description = "Capture RDS Event 0161 for ${each.value.id}" + name = "rds-event-rule-${each.value.id}-snapshot-created" + description = "Capture RDS Event 0042 (Snapshot Created) for ${each.value.id}" event_pattern = jsonencode({ - source = ["aws.rds"], - detail-type = ["RDS DB Instance Event"], - resources = [each.value.arn], + source = ["aws.rds"], detail = { - EventCategories = ["snapshot"], - SourceType = ["db-instance"], - Message = ["RDS-EVENT-0161"] + SourceArn = [{ + "prefix" : "arn:aws:rds:eu-west-2:120038763019:snapshot:sql-to-parquet" + }], + EventID = ["RDS-EVENT-0042"] } }) tags = var.tags } -resource "aws_cloudwatch_event_target" "rds_event_target" { +resource "aws_cloudwatch_event_target" "rds_snapshot_created_event_target" { for_each = { for instance in local.rds_instances : instance.id => instance } - rule = aws_cloudwatch_event_rule.rds_event_rule[each.key].name - arn = module.rds-to-s3-copier.lambda_function_arn + rule = aws_cloudwatch_event_rule.rds_snapshot_created_event_rule[each.key].name + arn = module.trigger_rds_snapshot_export.lambda_function_arn +} + +resource "aws_cloudwatch_event_rule" "rds_snapshot_exported_event_rule" { + for_each = { for instance in local.rds_instances : instance.id => instance } + + name = "rds-event-rule-${each.value.id}-snapshot-exported" + description = "Capture RDS Event 0161 (Snapshot Exported) for ${each.value.id}" + + event_pattern = jsonencode({ + source = ["aws.rds"], + detail = { + SourceArn = [{ + "prefix" : "arn:aws:rds:eu-west-2:120038763019:snapshot:sql-to-parquet" + }], + EventID = ["RDS-EVENT-0161"] + } + }) + + tags = var.tags +} + +resource "aws_cloudwatch_event_target" "rds_export_s3_to_s3_event_target" { + for_each = { for instance in local.rds_instances : instance.id => instance } + + rule = aws_cloudwatch_event_rule.rds_snapshot_exported_event_rule[each.key].name + arn = module.rds_snapshot_s3_to_s3_copier.lambda_function_arn } From 0aa9dc4ffd716cfe4ac4d54e3845612643c44b7b Mon Sep 17 00:00:00 2001 From: timburke-hackit Date: Thu, 12 Oct 2023 11:14:56 +0100 Subject: [PATCH 06/17] iam polices for lambdas --- terraform/modules/rds-snapshot-to-s3/iam.tf | 102 ++++++++++++++++++-- 1 file changed, 96 insertions(+), 6 deletions(-) diff --git a/terraform/modules/rds-snapshot-to-s3/iam.tf b/terraform/modules/rds-snapshot-to-s3/iam.tf index 4cf3f99a6..117b286bd 100644 --- a/terraform/modules/rds-snapshot-to-s3/iam.tf +++ b/terraform/modules/rds-snapshot-to-s3/iam.tf @@ -1,10 +1,19 @@ -resource "aws_lambda_permission" "allow_cloudwatch" { +resource "aws_lambda_permission" "allow_cloudwatch_snapshot_export_trigger" { for_each = { for instance in local.rds_instances : instance.id => instance } action = "lambda:InvokeFunction" - function_name = module.rds-to-s3-copier.lambda_function_arn + function_name = module.trigger_rds_snapshot_export.lambda_function_arn principal = "events.amazonaws.com" - source_arn = aws_cloudwatch_event_rule.rds_event_rule[each.key].arn + source_arn = aws_cloudwatch_event_rule.rds_snapshot_created_event_rule[each.key].arn +} + +resource "aws_lambda_permission" "allow_cloudwatch_snapshot_copier" { + for_each = { for instance in local.rds_instances : instance.id => instance } + + action = "lambda:InvokeFunction" + function_name = module.rds_snapshot_s3_to_s3_copier.lambda_function_arn + principal = "events.amazonaws.com" + source_arn = aws_cloudwatch_event_rule.rds_snapshot_exported_event_rule[each.key].arn } resource "aws_iam_role" "cloudwatch_events_role" { @@ -32,10 +41,91 @@ resource "aws_iam_role_policy" "cloudwatch_events_policy" { Version = "2012-10-17", Statement = [ { - Effect = "Allow", - Action = "lambda:InvokeFunction", - Resource = module.rds-to-s3-copier.lambda_function_arn, + Effect = "Allow", + Action = "lambda:InvokeFunction", + Resource = [ + module.trigger_rds_snapshot_export.lambda_function_arn, + module.rds_snapshot_s3_to_s3_copier.lambda_function_arn + ] } ] }) } + +resource "aws_iam_role" "rds_snapshot_to_s3_lambda_role" { + name = "rds-snapshot-to-s3-lambda-role" + assume_role_policy = jsondecode(data.aws_iam_policy_document.lambda_assume_role.json) + +} + +resource "aws_iam_policy_document" "rds_snapshot_to_s3_lambda" { + name = "rds-snapshot-to-s3-lambda-policy" + + statement { + actions = [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", + ] + effect = "Allow" + resources = [ + "*" + ] + } + + statement { + actions = [ + "rds:StartExportTask", + "rds:DescribeExportTasks" + ] + effect = "Allow" + resources = [ + local.rds_instances[*].arn + ] + } +} + +resource "aws_iam_policy_attachment" "name" { + name = "rds-snapshot-to-s3-lambda-policy-attachment" + policy_arn = aws_iam_policy.rds_snapshot_to_s3_lambda.arn + roles = [ + aws_iam_role.rds_snapshot_to_s3_lambda_role.name + ] +} + +resource "aws_iam_role" "rds_snapshot_s3_to_s3_copier_lambda_role" { + name = "rds-snapshot-s3-to-s3-copier-lambda-role" + assume_role_policy = jsondecode(data.aws_iam_policy_document.lambda_assume_role.json) +} + +resource "aws_iam_policy_document" "rds_snapshot_s3_to_s3_copier_role_policy" { + name = "rds-snapshot-s3-to-s3-copier-lambda-policy" + + statement { + actions = [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", + ] + effect = "Allow" + resources = [ + "*" + ] + } + + statement { + actions = [ + "s3:GetObject", + "s3:PutObject", + "s3:ListBucket", + "s3:DeleteObject" + ] + effect = "Allow" + resources = [ + module.landing_zone.bucket_arn, + "${module.landing_zone.bucket_arn}/*", + module.raw_zone.bucket_arn, + "${module.raw_zone.bucket_arn}/*" + ] + } +} From e24d6da83bef4368698a75195fce159008f1651f Mon Sep 17 00:00:00 2001 From: timburke-hackit Date: Mon, 16 Oct 2023 12:11:14 +0100 Subject: [PATCH 07/17] move rds storage to separate module --- terraform/core/10-aws-s3-buckets.tf | 11 +++++++++++ terraform/core/29-db-snapshot-to-s3.tf | 4 ++++ terraform/core/36-liberator-import.tf | 9 +++++++-- .../db-snapshot-to-s3/01-inputs-required.tf | 16 ++++++++++++++++ terraform/modules/db-snapshot-to-s3/10-s3.tf | 12 ------------ .../db-snapshot-to-s3/20-rds-to-s3-lambda.tf | 4 ++-- .../30-rds-snapshot-export-service.tf | 8 +++----- .../40-s3-to-s3-copier-lambda.tf | 8 ++++---- .../45-s3-to-s3-copier-queue.tf | 2 +- .../rds-snapshot-to-s3/01-inputs-required.tf | 11 ++++++++--- 10 files changed, 56 insertions(+), 29 deletions(-) diff --git a/terraform/core/10-aws-s3-buckets.tf b/terraform/core/10-aws-s3-buckets.tf index 3fea2d3ce..09cf05b10 100644 --- a/terraform/core/10-aws-s3-buckets.tf +++ b/terraform/core/10-aws-s3-buckets.tf @@ -424,3 +424,14 @@ resource "aws_s3_bucket_versioning" "ssl_connection_resources" { status = "Enabled" } } + +module "rds_export_storage" { + source = "../modules/s3-bucket" + + tags = module.tags.values + project = var.project + environment = var.environment + identifier_prefix = local.identifier_prefix + bucket_name = "RDS Export Storage" + bucket_identifier = "rds-export-storage" +} diff --git a/terraform/core/29-db-snapshot-to-s3.tf b/terraform/core/29-db-snapshot-to-s3.tf index cdc7dd33f..cd63f5ba6 100644 --- a/terraform/core/29-db-snapshot-to-s3.tf +++ b/terraform/core/29-db-snapshot-to-s3.tf @@ -23,6 +23,10 @@ module "db_snapshot_to_s3" { zone_kms_key_arn = module.raw_zone.kms_key_arn zone_bucket_arn = module.raw_zone.bucket_arn zone_bucket_id = module.raw_zone.bucket_id + rds_export_storage_bucket_arn = module.rds_export_storage.bucket_arn + rds_export_storage_bucket_id = module.rds_export_storage.bucket_id + rds_export_storage_kms_key_arn = module.rds_export_storage.kms_key_arn + rds_export_storage_kms_key_id = module.rds_export_storage.kms_key_id service_area = "unrestricted" rds_instance_ids = var.rds_instance_ids diff --git a/terraform/core/36-liberator-import.tf b/terraform/core/36-liberator-import.tf index b5dc2448a..9859abf00 100644 --- a/terraform/core/36-liberator-import.tf +++ b/terraform/core/36-liberator-import.tf @@ -37,6 +37,10 @@ module "liberator_db_snapshot_to_s3" { zone_kms_key_arn = module.landing_zone.kms_key_arn zone_bucket_arn = module.landing_zone.bucket_arn zone_bucket_id = module.landing_zone.bucket_id + rds_export_storage_bucket_arn = module.rds_export_storage.bucket_arn + rds_export_storage_bucket_id = module.rds_export_storage.bucket_id + rds_export_storage_kms_key_arn = module.rds_export_storage.kms_key_arn + rds_export_storage_kms_key_id = module.rds_export_storage.kms_key_id service_area = "parking" rds_instance_ids = [for item in module.liberator_dump_to_rds_snapshot : item.rds_instance_id] workflow_name = aws_glue_workflow.parking_liberator_data.name @@ -79,7 +83,7 @@ data "aws_iam_policy_document" "lambda_assume_role" { ### New modules for liberator ingestion module "liberator_rds_snapshot_to_s3" { - count = 0 + count = 1 source = "../modules/rds-snapshot-to-s3" tags = module.tags.values identifier_prefix = local.identifier_prefix @@ -87,8 +91,9 @@ module "liberator_rds_snapshot_to_s3" { environment = var.environment lambda_artefact_storage_bucket = module.lambda_artefact_storage.bucket_id zone_kms_key_arn = module.landing_zone.kms_key_arn - zone_bucket_arn = module.landing_zone.bucket_arn + source_bucket_arn = module.landing_zone.bucket_arn zone_bucket_id = module.landing_zone.bucket_id + target_bucket_arn = module.raw_zone.bucket_arn service_area = "parking" rds_instance_ids = [for item in module.liberator_dump_to_rds_snapshot : item.rds_instance_id] rds_instance_arns = [for item in module.liberator_dump_to_rds_snapshot : item.rds_instance_arn] diff --git a/terraform/modules/db-snapshot-to-s3/01-inputs-required.tf b/terraform/modules/db-snapshot-to-s3/01-inputs-required.tf index 03f89a14f..7fd0d2dcf 100644 --- a/terraform/modules/db-snapshot-to-s3/01-inputs-required.tf +++ b/terraform/modules/db-snapshot-to-s3/01-inputs-required.tf @@ -42,3 +42,19 @@ variable "service_area" { variable "rds_instance_ids" { type = list(string) } + +variable "rds_export_storage_bucket_arn" { + type = string +} + +variable "rds_export_storage_kms_key_arn" { + type = string +} + +variable "rds_export_storage_kms_key_id" { + type = string +} + +variable "rds_export_storage_bucket_id" { + type = string +} diff --git a/terraform/modules/db-snapshot-to-s3/10-s3.tf b/terraform/modules/db-snapshot-to-s3/10-s3.tf index 63c4812ac..8b1378917 100644 --- a/terraform/modules/db-snapshot-to-s3/10-s3.tf +++ b/terraform/modules/db-snapshot-to-s3/10-s3.tf @@ -1,13 +1 @@ -module "rds_export_storage" { - source = "../s3-bucket" - tags = var.tags - project = var.project - environment = var.environment - identifier_prefix = var.identifier_prefix - bucket_name = "RDS Export Storage" - bucket_identifier = "rds-export-storage${var.aws_account_suffix}" - role_arns_to_share_access_with = [ - aws_iam_role.rds_snapshot_to_s3_lambda.arn - ] -} diff --git a/terraform/modules/db-snapshot-to-s3/20-rds-to-s3-lambda.tf b/terraform/modules/db-snapshot-to-s3/20-rds-to-s3-lambda.tf index 71fef2490..f48dd59e3 100644 --- a/terraform/modules/db-snapshot-to-s3/20-rds-to-s3-lambda.tf +++ b/terraform/modules/db-snapshot-to-s3/20-rds-to-s3-lambda.tf @@ -159,8 +159,8 @@ resource "aws_lambda_function" "rds_snapshot_to_s3_lambda" { environment { variables = { IAM_ROLE_ARN = aws_iam_role.rds_snapshot_export_service.arn, - KMS_KEY_ID = module.rds_export_storage.kms_key_id, - S3_BUCKET_NAME = module.rds_export_storage.bucket_id, + KMS_KEY_ID = var.rds_export_storage_kms_key_id, + S3_BUCKET_NAME = var.rds_export_storage_bucket_id, COPIER_QUEUE_ARN = aws_sqs_queue.s3_to_s3_copier.arn } } diff --git a/terraform/modules/db-snapshot-to-s3/30-rds-snapshot-export-service.tf b/terraform/modules/db-snapshot-to-s3/30-rds-snapshot-export-service.tf index d836a7dc5..4072a87b8 100644 --- a/terraform/modules/db-snapshot-to-s3/30-rds-snapshot-export-service.tf +++ b/terraform/modules/db-snapshot-to-s3/30-rds-snapshot-export-service.tf @@ -41,10 +41,8 @@ data "aws_iam_policy_document" "rds_snapshot_export_service" { "s3:DeleteObject*" ] resources = [ - module.rds_export_storage.bucket_arn, - "${module.rds_export_storage.bucket_arn}/*", - module.rds_export_storage.bucket_arn, - "${module.rds_export_storage.bucket_arn}/*", + var.rds_export_storage_bucket_arn, + "${var.rds_export_storage_bucket_arn}/*" ] } @@ -54,7 +52,7 @@ data "aws_iam_policy_document" "rds_snapshot_export_service" { ] effect = "Allow" resources = [ - module.rds_export_storage.kms_key_arn + var.rds_export_storage_kms_key_arn ] } } diff --git a/terraform/modules/db-snapshot-to-s3/40-s3-to-s3-copier-lambda.tf b/terraform/modules/db-snapshot-to-s3/40-s3-to-s3-copier-lambda.tf index 6ec390879..f534dfc5d 100644 --- a/terraform/modules/db-snapshot-to-s3/40-s3-to-s3-copier-lambda.tf +++ b/terraform/modules/db-snapshot-to-s3/40-s3-to-s3-copier-lambda.tf @@ -53,8 +53,8 @@ data "aws_iam_policy_document" "s3_to_s3_copier_lambda" { ] effect = "Allow" resources = [ - module.rds_export_storage.kms_key_arn, - "${module.rds_export_storage.bucket_arn}/*", + var.rds_export_storage_bucket_arn, + "${var.rds_export_storage_bucket_arn}/*", var.zone_kms_key_arn, var.zone_bucket_arn, "${var.zone_bucket_arn}/*", @@ -73,8 +73,8 @@ data "aws_iam_policy_document" "s3_to_s3_copier_lambda" { resources = [ var.zone_bucket_arn, "${var.zone_bucket_arn}/*", - module.rds_export_storage.bucket_arn, - "${module.rds_export_storage.bucket_arn}/*" + var.rds_export_storage_bucket_arn, + "${var.rds_export_storage_bucket_arn}/*" ] } diff --git a/terraform/modules/db-snapshot-to-s3/45-s3-to-s3-copier-queue.tf b/terraform/modules/db-snapshot-to-s3/45-s3-to-s3-copier-queue.tf index dca7a064d..6a08a345e 100644 --- a/terraform/modules/db-snapshot-to-s3/45-s3-to-s3-copier-queue.tf +++ b/terraform/modules/db-snapshot-to-s3/45-s3-to-s3-copier-queue.tf @@ -13,7 +13,7 @@ resource "aws_sqs_queue" "s3_to_s3_copier" { visibility_timeout_seconds = local.lambda_timeout * 6 name = lower("${var.identifier_prefix}-s3-to-s3-copier") - kms_master_key_id = aws_kms_key.s3_to_s3_copier_kms_key.key_id + kms_master_key_id = var.rds_export_storage_kms_key_id } resource "aws_kms_key" "s3_to_s3_copier_kms_key" { diff --git a/terraform/modules/rds-snapshot-to-s3/01-inputs-required.tf b/terraform/modules/rds-snapshot-to-s3/01-inputs-required.tf index a7e3fe55d..3618f1ac5 100644 --- a/terraform/modules/rds-snapshot-to-s3/01-inputs-required.tf +++ b/terraform/modules/rds-snapshot-to-s3/01-inputs-required.tf @@ -26,9 +26,6 @@ variable "zone_kms_key_arn" { type = string } -variable "zone_bucket_arn" { - type = string -} variable "zone_bucket_id" { type = string @@ -46,3 +43,11 @@ variable "rds_instance_ids" { variable "rds_instance_arns" { type = list(string) } + +variable "source_bucket_arn" { + type = string +} + +variable "target_bucket_arn" { + type = string +} From d8b432612e2689066f1d39b2e6c73df0880fca9b Mon Sep 17 00:00:00 2001 From: timburke-hackit Date: Mon, 16 Oct 2023 12:11:47 +0100 Subject: [PATCH 08/17] comment out test module --- .../core/29-db-snapshot-to-s3-sandbox.tf | 60 ++++++++++--------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/terraform/core/29-db-snapshot-to-s3-sandbox.tf b/terraform/core/29-db-snapshot-to-s3-sandbox.tf index f3333f289..920a6243e 100644 --- a/terraform/core/29-db-snapshot-to-s3-sandbox.tf +++ b/terraform/core/29-db-snapshot-to-s3-sandbox.tf @@ -2,13 +2,13 @@ # 1. Deploy this first to get the database and bastion host in place module "db_snapshot_to_s3_sandbox_resources" { - count = 0 - source = "../modules/db-snapshot-to-s3-sandbox-resources" - tags = module.tags.values - identifier_prefix = local.identifier_prefix - aws_sandbox_subnet_ids = var.aws_sandbox_subnet_ids - aws_sandbox_account_id = var.aws_sandbox_account_id - aws_sandbox_vpc_id = var.aws_sandbox_vpc_id + count = 0 + source = "../modules/db-snapshot-to-s3-sandbox-resources" + tags = module.tags.values + identifier_prefix = local.identifier_prefix + aws_sandbox_subnet_ids = var.aws_sandbox_subnet_ids + aws_sandbox_account_id = var.aws_sandbox_account_id + aws_sandbox_vpc_id = var.aws_sandbox_vpc_id providers = { aws = aws @@ -20,7 +20,7 @@ module "db_snapshot_to_s3_sandbox_resources" { # 3. lambda_artefact_storage_for_sandbox_account and db_snapshot_to_s3_sandbox can be deployed at the same time module "lambda_artefact_storage_for_sandbox_account" { - count = 0 + count = 0 source = "../modules/s3-bucket" tags = module.tags.values project = var.project @@ -34,25 +34,27 @@ module "lambda_artefact_storage_for_sandbox_account" { } } -module "db_snapshot_to_s3_sandbox" { - count = 0 - source = "../modules/db-snapshot-to-s3" - tags = module.tags.values - project = var.project - environment = var.environment - identifier_prefix = local.identifier_prefix - lambda_artefact_storage_bucket = module.lambda_artefact_storage_for_sandbox_account[0].bucket_id - zone_kms_key_arn = module.raw_zone.kms_key_arn - zone_bucket_arn = module.raw_zone.bucket_arn - zone_bucket_id = module.raw_zone.bucket_id - service_area = "unrestricted" - rds_instance_ids = var.rds_instance_ids - aws_account_suffix = "-sandbox" - - providers = { - aws = aws.aws_sandbox_account - } -} +#module "db_snapshot_to_s3_sandbox" { +# count = 0 +# source = "../modules/db-snapshot-to-s3" +# tags = module.tags.values +# project = var.project +# environment = var.environment +# identifier_prefix = local.identifier_prefix +# lambda_artefact_storage_bucket = module.lambda_artefact_storage_for_sandbox_account[0].bucket_id +# zone_kms_key_arn = module.raw_zone.kms_key_arn +# zone_bucket_arn = module.raw_zone.bucket_arn +# zone_bucket_id = module.raw_zone.bucket_id +# rds_export_storage_bucket_arn = module.rds_export_storage.bucket_arn +# rds_export_storage_kms_key_arn = module.rds_export_storage.kms_key_arn +# service_area = "unrestricted" +# rds_instance_ids = var.rds_instance_ids +# aws_account_suffix = "-sandbox" +# +# providers = { +# aws = aws.aws_sandbox_account +# } +#} #4. Update the raw zone bucket on DP dev account in your workspace with the following bucket and bucket key statements # Use these as inputs for bucket_policy_statements and bucket_key_policy_statements in the raw zone bucket module @@ -60,7 +62,7 @@ module "db_snapshot_to_s3_sandbox" { # sandbox_s3_to_s3_copier_write_access_to_raw_zone_statement = { # sid = "AllowSandboxS3toS3CopierWriteAccessToRawZoneUnrestrictedLocation" # effect = "Allow" - + # actions = [ # "s3:ListBucket", # "s3:PutObject", @@ -95,7 +97,7 @@ module "db_snapshot_to_s3_sandbox" { # "arn:aws:iam::${var.aws_sandbox_account_id}:role/${local.identifier_prefix}-s3-to-s3-copier-lambda" # ] # } - + # } #5. Uncomment the statement in the sandbox database key policy to allow the rds snapshot to s3 lambda role access to the key. This must be done after all other resources have been deployed. From dbab932c499c9e09011e0132e958c9f69dc48291 Mon Sep 17 00:00:00 2001 From: timburke-hackit Date: Mon, 16 Oct 2023 12:12:12 +0100 Subject: [PATCH 09/17] output lambda role --- terraform/modules/rds-snapshot-to-s3/99-outputs.tf | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/terraform/modules/rds-snapshot-to-s3/99-outputs.tf b/terraform/modules/rds-snapshot-to-s3/99-outputs.tf index 8b1378917..da7ee1b28 100644 --- a/terraform/modules/rds-snapshot-to-s3/99-outputs.tf +++ b/terraform/modules/rds-snapshot-to-s3/99-outputs.tf @@ -1 +1,4 @@ - +output "rds_snapshot_s3_to_s3_copier_lambda_role_arn" { + description = "ARN for the s3_to_s3_copier_lambda_role" + value = aws_iam_role.rds_snapshot_s3_to_s3_copier_lambda_role.arn +} From eb1fd0dfafebe60840c58a4ec080a95d773116de Mon Sep 17 00:00:00 2001 From: timburke-hackit Date: Mon, 16 Oct 2023 12:12:33 +0100 Subject: [PATCH 10/17] lambda policies --- terraform/modules/rds-snapshot-to-s3/iam.tf | 52 ++++++++++++++------- 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/terraform/modules/rds-snapshot-to-s3/iam.tf b/terraform/modules/rds-snapshot-to-s3/iam.tf index 117b286bd..ce9db3715 100644 --- a/terraform/modules/rds-snapshot-to-s3/iam.tf +++ b/terraform/modules/rds-snapshot-to-s3/iam.tf @@ -58,9 +58,7 @@ resource "aws_iam_role" "rds_snapshot_to_s3_lambda_role" { } -resource "aws_iam_policy_document" "rds_snapshot_to_s3_lambda" { - name = "rds-snapshot-to-s3-lambda-policy" - +data "aws_iam_policy_document" "rds_snapshot_to_s3_lambda" { statement { actions = [ "logs:CreateLogGroup", @@ -85,22 +83,14 @@ resource "aws_iam_policy_document" "rds_snapshot_to_s3_lambda" { } } -resource "aws_iam_policy_attachment" "name" { - name = "rds-snapshot-to-s3-lambda-policy-attachment" - policy_arn = aws_iam_policy.rds_snapshot_to_s3_lambda.arn - roles = [ - aws_iam_role.rds_snapshot_to_s3_lambda_role.name - ] -} + resource "aws_iam_role" "rds_snapshot_s3_to_s3_copier_lambda_role" { name = "rds-snapshot-s3-to-s3-copier-lambda-role" assume_role_policy = jsondecode(data.aws_iam_policy_document.lambda_assume_role.json) } -resource "aws_iam_policy_document" "rds_snapshot_s3_to_s3_copier_role_policy" { - name = "rds-snapshot-s3-to-s3-copier-lambda-policy" - +data "aws_iam_policy_document" "rds_snapshot_s3_to_s3_copier_role_policy" { statement { actions = [ "logs:CreateLogGroup", @@ -122,10 +112,38 @@ resource "aws_iam_policy_document" "rds_snapshot_s3_to_s3_copier_role_policy" { ] effect = "Allow" resources = [ - module.landing_zone.bucket_arn, - "${module.landing_zone.bucket_arn}/*", - module.raw_zone.bucket_arn, - "${module.raw_zone.bucket_arn}/*" + var.source_bucket_arn, + "${var.source_bucket_arn}/*", + var.target_bucket_arn, + "${var.target_bucket_arn}/*" + ] + } +} + +resource "aws_iam_policy" "rds_snapshot_s3_to_s3_copier_role_policy" { + name = lower("${var.identifier_prefix}-rds-snapshot-s3-to-s3-copier-lambda-policy") + policy = data.aws_iam_policy_document.rds_snapshot_s3_to_s3_copier_role_policy.json + tags = var.tags +} + +resource "aws_iam_policy_attachment" "rds_snapshot_copier_attachment" { + name = "${var.identifier_prefix}-rds-snapshot-s3-to-s3-lambda-policy-attachment" + policy_arn = aws_iam_policy.rds_snapshot_s3_to_s3_copier_role_policy.arn + roles = [ + aws_iam_role.rds_snapshot_to_s3_lambda_role.name + ] +} + +data "aws_iam_policy_document" "lambda_assume_role" { + statement { + actions = [ + "sts:AssumeRole" ] + principals { + identifiers = [ + "lambda.amazonaws.com" + ] + type = "Service" + } } } From 62b4efdbda372bfc39cde383c3576296bf98d470 Mon Sep 17 00:00:00 2001 From: timburke-hackit Date: Mon, 16 Oct 2023 12:46:22 +0100 Subject: [PATCH 11/17] remove unnecessary jsonencode() --- terraform/modules/rds-snapshot-to-s3/iam.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/modules/rds-snapshot-to-s3/iam.tf b/terraform/modules/rds-snapshot-to-s3/iam.tf index ce9db3715..77c80e4e2 100644 --- a/terraform/modules/rds-snapshot-to-s3/iam.tf +++ b/terraform/modules/rds-snapshot-to-s3/iam.tf @@ -54,7 +54,7 @@ resource "aws_iam_role_policy" "cloudwatch_events_policy" { resource "aws_iam_role" "rds_snapshot_to_s3_lambda_role" { name = "rds-snapshot-to-s3-lambda-role" - assume_role_policy = jsondecode(data.aws_iam_policy_document.lambda_assume_role.json) + assume_role_policy = data.aws_iam_policy_document.lambda_assume_role.json } @@ -78,7 +78,7 @@ data "aws_iam_policy_document" "rds_snapshot_to_s3_lambda" { ] effect = "Allow" resources = [ - local.rds_instances[*].arn + local.rds_instances[0].arn ] } } @@ -87,7 +87,7 @@ data "aws_iam_policy_document" "rds_snapshot_to_s3_lambda" { resource "aws_iam_role" "rds_snapshot_s3_to_s3_copier_lambda_role" { name = "rds-snapshot-s3-to-s3-copier-lambda-role" - assume_role_policy = jsondecode(data.aws_iam_policy_document.lambda_assume_role.json) + assume_role_policy = data.aws_iam_policy_document.lambda_assume_role.json } data "aws_iam_policy_document" "rds_snapshot_s3_to_s3_copier_role_policy" { From b8647b6f35adf9933d4fab2b5b5bdca0c162de9b Mon Sep 17 00:00:00 2001 From: timburke-hackit Date: Mon, 16 Oct 2023 12:46:33 +0100 Subject: [PATCH 12/17] fix paths --- terraform/modules/rds-snapshot-to-s3/lambda.tf | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/terraform/modules/rds-snapshot-to-s3/lambda.tf b/terraform/modules/rds-snapshot-to-s3/lambda.tf index 14f20ae54..b54330216 100644 --- a/terraform/modules/rds-snapshot-to-s3/lambda.tf +++ b/terraform/modules/rds-snapshot-to-s3/lambda.tf @@ -1,7 +1,7 @@ module "trigger_rds_snapshot_export" { source = "../aws-lambda" lambda_name = "export-rds-snapshot-to-s3" - runtime = "python3.10" + runtime = "python3.9" handler = "lambda_function.lambda_handler" lambda_artefact_storage_bucket = var.lambda_artefact_storage_bucket lambda_source_dir = "../../lambdas/export_rds_snapshot_to_s3" @@ -14,12 +14,12 @@ module "trigger_rds_snapshot_export" { module "rds_snapshot_s3_to_s3_copier" { source = "../aws-lambda" lambda_name = "rds-export-s3-to-s3-copier" - runtime = "python3.10" + runtime = "python3.9" handler = "lambda_function.lambda_handler" lambda_artefact_storage_bucket = var.lambda_artefact_storage_bucket - lambda_source_dir = "../../lambdas/rds_export_s3_to_s3_copier" - lambda_output_path = "../../lambdas/rds-export-s3-to-s3-copier.zip" + lambda_source_dir = "../../lambdas/rds_snapshot_export_s3_to_s3_copier" + lambda_output_path = "../../lambdas/rds_snapshot_export_s3_to_s3_copier.zip" s3_key = "rds-export-s3-to-s3-copier.zip" identifier_prefix = var.identifier_prefix tags = var.tags -} \ No newline at end of file +} From c6bd2a03d8952c6b49695e3924a5a85e2b479518 Mon Sep 17 00:00:00 2001 From: timburke-hackit Date: Mon, 16 Oct 2023 13:52:13 +0100 Subject: [PATCH 13/17] add sse for rds export bucket --- terraform/core/10-aws-s3-buckets.tf | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/terraform/core/10-aws-s3-buckets.tf b/terraform/core/10-aws-s3-buckets.tf index 09cf05b10..3f116508a 100644 --- a/terraform/core/10-aws-s3-buckets.tf +++ b/terraform/core/10-aws-s3-buckets.tf @@ -435,3 +435,14 @@ module "rds_export_storage" { bucket_name = "RDS Export Storage" bucket_identifier = "rds-export-storage" } + +resource "aws_s3_bucket_server_side_encryption_configuration" "rds_export_storage_encryption" { + bucket = module.rds_export_storage.bucket_id + + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "aws:kms" + } + bucket_key_enabled = true + } +} From 3cd824d58b36da35e330ffcbfcd472a2060fbafd Mon Sep 17 00:00:00 2001 From: timburke-hackit Date: Mon, 16 Oct 2023 14:18:43 +0100 Subject: [PATCH 14/17] re-add existing rds s3 storage bucket --- terraform/modules/db-snapshot-to-s3/10-s3.tf | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/terraform/modules/db-snapshot-to-s3/10-s3.tf b/terraform/modules/db-snapshot-to-s3/10-s3.tf index 8b1378917..63c4812ac 100644 --- a/terraform/modules/db-snapshot-to-s3/10-s3.tf +++ b/terraform/modules/db-snapshot-to-s3/10-s3.tf @@ -1 +1,13 @@ +module "rds_export_storage" { + source = "../s3-bucket" + tags = var.tags + project = var.project + environment = var.environment + identifier_prefix = var.identifier_prefix + bucket_name = "RDS Export Storage" + bucket_identifier = "rds-export-storage${var.aws_account_suffix}" + role_arns_to_share_access_with = [ + aws_iam_role.rds_snapshot_to_s3_lambda.arn + ] +} From 34bd21f729c317f2d25dbf66303f5e38439fff8d Mon Sep 17 00:00:00 2001 From: timburke-hackit Date: Mon, 16 Oct 2023 14:33:06 +0100 Subject: [PATCH 15/17] disable sqs lambda event source mapping --- terraform/modules/db-snapshot-to-s3/25-rds-to-s3-queue.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/modules/db-snapshot-to-s3/25-rds-to-s3-queue.tf b/terraform/modules/db-snapshot-to-s3/25-rds-to-s3-queue.tf index 0a73da849..0d6779b98 100644 --- a/terraform/modules/db-snapshot-to-s3/25-rds-to-s3-queue.tf +++ b/terraform/modules/db-snapshot-to-s3/25-rds-to-s3-queue.tf @@ -128,7 +128,7 @@ resource "aws_sns_topic_subscription" "subscribe_sqs_to_sns_topic" { resource "aws_lambda_event_source_mapping" "event_source_mapping" { event_source_arn = aws_sqs_queue.rds_snapshot_to_s3.arn - enabled = true + enabled = false function_name = aws_lambda_function.rds_snapshot_to_s3_lambda.arn batch_size = 1 } From c91bf5c4b1f791c99c99b0927483410f3bc3a32d Mon Sep 17 00:00:00 2001 From: timburke-hackit Date: Mon, 16 Oct 2023 15:11:43 +0100 Subject: [PATCH 16/17] add environment vars --- terraform/core/36-liberator-import.tf | 2 ++ .../modules/rds-snapshot-to-s3/02-inputs-optional.tf | 12 ++++++++++++ terraform/modules/rds-snapshot-to-s3/lambda.tf | 7 +++++++ 3 files changed, 21 insertions(+) diff --git a/terraform/core/36-liberator-import.tf b/terraform/core/36-liberator-import.tf index 9859abf00..693828b80 100644 --- a/terraform/core/36-liberator-import.tf +++ b/terraform/core/36-liberator-import.tf @@ -94,6 +94,8 @@ module "liberator_rds_snapshot_to_s3" { source_bucket_arn = module.landing_zone.bucket_arn zone_bucket_id = module.landing_zone.bucket_id target_bucket_arn = module.raw_zone.bucket_arn + source_prefix = "parking/liberator/" + target_prefix = "parking/liberator/" service_area = "parking" rds_instance_ids = [for item in module.liberator_dump_to_rds_snapshot : item.rds_instance_id] rds_instance_arns = [for item in module.liberator_dump_to_rds_snapshot : item.rds_instance_arn] diff --git a/terraform/modules/rds-snapshot-to-s3/02-inputs-optional.tf b/terraform/modules/rds-snapshot-to-s3/02-inputs-optional.tf index 86faba8a8..2dadb940f 100644 --- a/terraform/modules/rds-snapshot-to-s3/02-inputs-optional.tf +++ b/terraform/modules/rds-snapshot-to-s3/02-inputs-optional.tf @@ -27,3 +27,15 @@ variable "aws_account_suffix" { type = string default = "" } + +variable "source_prefix" { + description = "Prefix to be used for the source bucket location" + type = string + default = "" +} + +variable "target_prefix" { + description = "Prefix to be used for the target bucket location" + type = string + default = "" +} diff --git a/terraform/modules/rds-snapshot-to-s3/lambda.tf b/terraform/modules/rds-snapshot-to-s3/lambda.tf index b54330216..1731010ae 100644 --- a/terraform/modules/rds-snapshot-to-s3/lambda.tf +++ b/terraform/modules/rds-snapshot-to-s3/lambda.tf @@ -22,4 +22,11 @@ module "rds_snapshot_s3_to_s3_copier" { s3_key = "rds-export-s3-to-s3-copier.zip" identifier_prefix = var.identifier_prefix tags = var.tags + environment_variables = { + "SOURCE_BUCKET" = var.source_bucket_arn + "TARGET_BUCKET" = var.target_bucket_arn + "SOURCE_PREFIX" = var.source_prefix + "TARGET_PREFIX" = var.target_prefix + "WORKFLOW_NAME" = var.workflow_name + } } From 72404c8bf42244a96a968c34d06ee6c96b487c46 Mon Sep 17 00:00:00 2001 From: timburke-hackit Date: Mon, 16 Oct 2023 15:24:10 +0100 Subject: [PATCH 17/17] more env vars --- terraform/modules/rds-snapshot-to-s3/lambda.tf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/terraform/modules/rds-snapshot-to-s3/lambda.tf b/terraform/modules/rds-snapshot-to-s3/lambda.tf index 1731010ae..ddb0ed9fb 100644 --- a/terraform/modules/rds-snapshot-to-s3/lambda.tf +++ b/terraform/modules/rds-snapshot-to-s3/lambda.tf @@ -9,6 +9,11 @@ module "trigger_rds_snapshot_export" { s3_key = "export-rds-snapshot-to-s3.zip" identifier_prefix = var.identifier_prefix tags = var.tags + environment_variables = { + "BUCKET_NAME" = var.zone_bucket_id + "IAM_ROLE_ARN" = aws_iam_role.rds_snapshot_to_s3_lambda_role.arn + "KMS_KEY_ID" = var.zone_kms_key_arn + } } module "rds_snapshot_s3_to_s3_copier" {