update variables and policies (#1474)

* clarify variables

* export role policy update

* update export bucket var

* kms permissions for copier

* update env variables

* workflow permissions

* add policy params for rds snapshot

* update lambda handler

* add lambda timeouts

* update s3 prefix

terraform/core/36-liberator-import.tf

@@ -90,12 +90,16 @@ module "liberator_rds_snapshot_to_s3" {
   project                        = var.project
   environment                    = var.environment
   lambda_artefact_storage_bucket = module.lambda_artefact_storage.bucket_id
-  zone_kms_key_arn               = module.landing_zone.kms_key_arn
-  source_bucket_arn              = module.landing_zone.bucket_arn
-  zone_bucket_id                 = module.landing_zone.bucket_id
-  target_bucket_arn              = module.raw_zone.bucket_arn
-  source_prefix                  = "parking/liberator/"
-  target_prefix                  = "parking/liberator/"
+  rds_export_bucket_arn          = module.rds_export_storage.bucket_arn
+  rds_export_bucket_id           = module.rds_export_storage.bucket_id
+  rds_export_storage_kms_key_arn = module.rds_export_storage.kms_key_arn
+  rds_export_storage_kms_key_id  = module.rds_export_storage.kms_key_id
+  rds_snapshot_service_arn       = module.liberator_db_snapshot_to_s3[0].rds_snapshot_service_arn
+  target_bucket_arn              = module.landing_zone.bucket_arn
+  target_bucket_id               = module.landing_zone.bucket_id
+  target_bucket_kms_key_arn      = module.landing_zone.kms_key_arn
+  target_bucket_kms_key_id       = module.landing_zone.kms_key_id
+  target_prefix                  = "parking"
   service_area                   = "parking"
   rds_instance_ids               = [for item in module.liberator_dump_to_rds_snapshot : item.rds_instance_id]
   rds_instance_arns              = [for item in module.liberator_dump_to_rds_snapshot : item.rds_instance_arn]

terraform/modules/db-snapshot-to-s3/99-outputs.tf

@@ -3,3 +3,8 @@ output "s3_to_s3_copier_lambda_role_arn" {
   description = "KMS Key arn"
   value       = aws_iam_role.s3_to_s3_copier_lambda.arn
 }
+
+output "rds_snapshot_service_arn" {
+  description = "RDS Snapshot Service ARN"
+  value       = aws_iam_role.rds_snapshot_export_service.arn
+}

terraform/modules/rds-snapshot-to-s3/01-inputs-required.tf

@@ -22,12 +22,10 @@ variable "lambda_artefact_storage_bucket" {
   type = string
 }
 
-variable "zone_kms_key_arn" {
+variable "target_bucket_kms_key_arn" {
   type = string
 }
-
-
-variable "zone_bucket_id" {
+variable "rds_export_bucket_id" {
   type = string
 }
 
@@ -44,10 +42,30 @@ variable "rds_instance_arns" {
   type = list(string)
 }
 
-variable "source_bucket_arn" {
+variable "rds_export_bucket_arn" {
   type = string
 }
 
 variable "target_bucket_arn" {
   type = string
 }
+
+variable "target_bucket_id" {
+  type = string
+}
+
+variable "rds_export_storage_kms_key_arn" {
+  type = string
+}
+
+variable "rds_export_storage_kms_key_id" {
+  type = string
+}
+
+variable "target_bucket_kms_key_id" {
+  type = string
+}
+
+variable "rds_snapshot_service_arn" {
+  type = string
+}

terraform/modules/rds-snapshot-to-s3/iam.tf

@@ -71,14 +71,32 @@ data "aws_iam_policy_document" "rds_snapshot_to_s3_lambda" {
     ]
   }
 
+  statement {
+    actions   = ["iam:PassRole"]
+    effect    = "Allow"
+    resources = [var.rds_snapshot_service_arn]
+  }
+
   statement {
     actions = [
       "rds:StartExportTask",
       "rds:DescribeExportTasks"
     ]
     effect = "Allow"
     resources = [
-      local.rds_instances[0].arn
+      "*"
+    ]
+  }
+
+  statement {
+    sid = "AllowKMSDecrypt"
+    actions = [
+      "kms:Decrypt",
+      "kms:GenerateDataKey*"
+    ]
+    effect = "Allow"
+    resources = [
+      var.rds_export_storage_kms_key_arn
     ]
   }
 }
@@ -112,12 +130,36 @@ data "aws_iam_policy_document" "rds_snapshot_s3_to_s3_copier_role_policy" {
     ]
     effect = "Allow"
     resources = [
-      var.source_bucket_arn,
-      "${var.source_bucket_arn}/*",
+      var.rds_export_bucket_arn,
+      "${var.rds_export_bucket_arn}/*",
       var.target_bucket_arn,
       "${var.target_bucket_arn}/*"
     ]
   }
+
+  statement {
+    actions = [
+      "kms:Decrypt",
+      "kms:DescribeKey",
+      "kms:GenerateDataKey"
+    ]
+    effect = "Allow"
+    resources = [
+      var.rds_export_storage_kms_key_arn,
+      var.target_bucket_kms_key_arn
+    ]
+  }
+
+  statement {
+    actions = [
+      "glue:StartWorkflowRun"
+    ]
+    effect = "Allow"
+    resources = [
+      var.workflow_arn,
+      var.backdated_workflow_arn
+    ]
+  }
 }
 
 resource "aws_iam_policy" "rds_snapshot_s3_to_s3_copier_role_policy" {

terraform/modules/rds-snapshot-to-s3/lambda.tf

@@ -2,34 +2,36 @@ module "trigger_rds_snapshot_export" {
   source                         = "../aws-lambda"
   lambda_name                    = "export-rds-snapshot-to-s3"
   runtime                        = "python3.9"
-  handler                        = "lambda_function.lambda_handler"
+  handler                        = "main.lambda_handler"
+  lambda_timeout                 = 60
   lambda_artefact_storage_bucket = var.lambda_artefact_storage_bucket
   lambda_source_dir              = "../../lambdas/export_rds_snapshot_to_s3"
   lambda_output_path             = "../../lambdas/export-rds-snapshot-to-s3.zip"
   s3_key                         = "export-rds-snapshot-to-s3.zip"
   identifier_prefix              = var.identifier_prefix
   tags                           = var.tags
   environment_variables = {
-    "BUCKET_NAME"  = var.zone_bucket_id
+    "BUCKET_NAME"  = var.rds_export_bucket_id
     "IAM_ROLE_ARN" = aws_iam_role.rds_snapshot_to_s3_lambda_role.arn
-    "KMS_KEY_ID"   = var.zone_kms_key_arn
+    "KMS_KEY_ID"   = var.rds_export_storage_kms_key_id
   }
 }
 
 module "rds_snapshot_s3_to_s3_copier" {
   source                         = "../aws-lambda"
   lambda_name                    = "rds-export-s3-to-s3-copier"
   runtime                        = "python3.9"
-  handler                        = "lambda_function.lambda_handler"
+  handler                        = "main.lambda_handler"
+  lambda_timeout                 = 900
   lambda_artefact_storage_bucket = var.lambda_artefact_storage_bucket
   lambda_source_dir              = "../../lambdas/rds_snapshot_export_s3_to_s3_copier"
   lambda_output_path             = "../../lambdas/rds_snapshot_export_s3_to_s3_copier.zip"
   s3_key                         = "rds-export-s3-to-s3-copier.zip"
   identifier_prefix              = var.identifier_prefix
   tags                           = var.tags
   environment_variables = {
-    "SOURCE_BUCKET" = var.source_bucket_arn
-    "TARGET_BUCKET" = var.target_bucket_arn
+    "SOURCE_BUCKET" = var.rds_export_bucket_id
+    "TARGET_BUCKET" = var.target_bucket_id
     "SOURCE_PREFIX" = var.source_prefix
     "TARGET_PREFIX" = var.target_prefix
     "WORKFLOW_NAME" = var.workflow_name