From 20a605fd554eea4ba59b52ed6907536fc7841955 Mon Sep 17 00:00:00 2001 From: timburke-hackit Date: Thu, 2 Nov 2023 10:01:37 +0000 Subject: [PATCH 1/8] iam:PassRole statements --- terraform/modules/rds-snapshot-to-s3/iam.tf | 22 ++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/terraform/modules/rds-snapshot-to-s3/iam.tf b/terraform/modules/rds-snapshot-to-s3/iam.tf index 96028cee5..bd59ea32f 100644 --- a/terraform/modules/rds-snapshot-to-s3/iam.tf +++ b/terraform/modules/rds-snapshot-to-s3/iam.tf @@ -64,6 +64,18 @@ data "aws_iam_policy_document" "lambda_assume_role" { type = "Service" } } + + statement { + actions = [ + "sts:AssumeRole" + ] + principals { + identifiers = [ + "rds.amazonaws.com" + ] + type = "Service" + } + } } # RDS Snapshot to S3 lambda IAM @@ -86,9 +98,13 @@ data "aws_iam_policy_document" "rds_snapshot_to_s3_lambda" { } statement { - actions = ["iam:PassRole"] - effect = "Allow" - resources = [var.rds_snapshot_service_arn] + actions = [ + "iam:PassRole" + ] + effect = "Allow" + resources = [ + aws_iam_role.rds_snapshot_to_s3_lambda_role + ] } statement { From bc6fabceee99ed2265e476d4f96f6249563e5ce9 Mon Sep 17 00:00:00 2001 From: timburke-hackit Date: Thu, 2 Nov 2023 10:13:13 +0000 Subject: [PATCH 2/8] update export principal --- terraform/modules/rds-snapshot-to-s3/iam.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/modules/rds-snapshot-to-s3/iam.tf b/terraform/modules/rds-snapshot-to-s3/iam.tf index bd59ea32f..b6313df17 100644 --- a/terraform/modules/rds-snapshot-to-s3/iam.tf +++ b/terraform/modules/rds-snapshot-to-s3/iam.tf @@ -71,7 +71,7 @@ data "aws_iam_policy_document" "lambda_assume_role" { ] principals { identifiers = [ - "rds.amazonaws.com" + "export.rds.amazonaws.com" ] type = "Service" } From 3cff6aa8056ea757628d121c9fe82a0e22b8995f Mon Sep 17 00:00:00 2001 From: timburke-hackit Date: Thu, 2 Nov 2023 10:14:31 +0000 Subject: [PATCH 3/8] kms actions --- terraform/modules/rds-snapshot-to-s3/iam.tf | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/terraform/modules/rds-snapshot-to-s3/iam.tf b/terraform/modules/rds-snapshot-to-s3/iam.tf index b6313df17..7b83e0f30 100644 --- a/terraform/modules/rds-snapshot-to-s3/iam.tf +++ b/terraform/modules/rds-snapshot-to-s3/iam.tf @@ -121,8 +121,15 @@ data "aws_iam_policy_document" "rds_snapshot_to_s3_lambda" { statement { sid = "AllowKMSDecrypt" actions = [ + "kms:Encrypt", + "kms:GenerateDataKey", "kms:Decrypt", - "kms:GenerateDataKey*" + "kms:GenerateDataKeyPairWithoutPlaintext", + "kms:ReEncryptFrom", + "kms:ReEncryptTo", + "kms:CreateGrant", + "kms:DescribeKey", + "kms:RetireGrant" ] effect = "Allow" resources = [ From 1dd0f3c0c15188658a41f5542749f8112ff88b96 Mon Sep 17 00:00:00 2001 From: timburke-hackit Date: Thu, 2 Nov 2023 11:14:56 +0000 Subject: [PATCH 4/8] read event to dict --- lambdas/export_rds_snapshot_to_s3/main.py | 6 ++++-- lambdas/rds_snapshot_export_s3_to_s3_copier/main.py | 11 +++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/lambdas/export_rds_snapshot_to_s3/main.py b/lambdas/export_rds_snapshot_to_s3/main.py index 50cedd42c..a4a762a35 100644 --- a/lambdas/export_rds_snapshot_to_s3/main.py +++ b/lambdas/export_rds_snapshot_to_s3/main.py @@ -9,12 +9,14 @@ def lambda_handler(event, context): - snapshot_identifier = event["detail"]["SnapshotIdentifier"] - source_arn = event["detail"]["SourceArn"] bucket_name = os.environ["BUCKET_NAME"] iam_role_arn = os.environ["IAM_ROLE_ARN"] kms_key_id = os.environ["KMS_KEY_ID"] + event = ast.literal_eval(event) + snapshot_identifier = event["detail"]["SourceIdentifier"] + source_arn = event["detail"]["SourceArn"] + try: rds.start_export_task( ExportTaskIdentifier=snapshot_identifier, diff --git a/lambdas/rds_snapshot_export_s3_to_s3_copier/main.py b/lambdas/rds_snapshot_export_s3_to_s3_copier/main.py index 6375a17b4..80a6a9771 100644 --- a/lambdas/rds_snapshot_export_s3_to_s3_copier/main.py +++ b/lambdas/rds_snapshot_export_s3_to_s3_copier/main.py @@ -1,5 +1,5 @@ import os - +import ast import boto3 @@ -57,8 +57,8 @@ def s3_copy_folder( continue source_key_split = source_key.split("/") parquet_file_name = source_key_split[-1] - database_name = source_key_split("/")[1] - table_name = source_key_split("/")[2] + database_name = source_key_split[1] + table_name = source_key_split[2] copy_object_params = { "Bucket": target_bucket, "CopySource": f"{source_bucket}/{source_key}", @@ -85,6 +85,8 @@ def start_workflow_run(workflow_name: str, glue_client): def lambda_handler(event, context) -> None: + print("## EVENT") + print(event) s3 = boto3.client("s3") source_bucket = os.environ["SOURCE_BUCKET"] @@ -98,7 +100,8 @@ def lambda_handler(event, context) -> None: else: target_prefix = "" - snapshot_id = event["detail"]["SnapshotIdentifier"] + event = ast.literal_eval(event) + snapshot_id = event["detail"]["SourceIdentifier"] s3_copy_folder( s3, source_bucket, source_prefix, target_bucket, target_prefix, snapshot_id From 73cd997d4e819daea8ebe5cb88d37f8e70cedcdd Mon Sep 17 00:00:00 2001 From: timburke-hackit Date: Thu, 2 Nov 2023 11:17:11 +0000 Subject: [PATCH 5/8] import ast --- lambdas/export_rds_snapshot_to_s3/main.py | 1 + lambdas/rds_snapshot_export_s3_to_s3_copier/main.py | 1 + 2 files changed, 2 insertions(+) diff --git a/lambdas/export_rds_snapshot_to_s3/main.py b/lambdas/export_rds_snapshot_to_s3/main.py index a4a762a35..0f9fda09c 100644 --- a/lambdas/export_rds_snapshot_to_s3/main.py +++ b/lambdas/export_rds_snapshot_to_s3/main.py @@ -1,5 +1,6 @@ import logging import os +import ast import boto3 from botocore.exceptions import ClientError diff --git a/lambdas/rds_snapshot_export_s3_to_s3_copier/main.py b/lambdas/rds_snapshot_export_s3_to_s3_copier/main.py index 80a6a9771..f62067a1c 100644 --- a/lambdas/rds_snapshot_export_s3_to_s3_copier/main.py +++ b/lambdas/rds_snapshot_export_s3_to_s3_copier/main.py @@ -1,5 +1,6 @@ import os import ast + import boto3 From e411e560ed0fa4f60cdd8b1eb800365b5b51bdc4 Mon Sep 17 00:00:00 2001 From: timburke-hackit Date: Thu, 2 Nov 2023 11:19:26 +0000 Subject: [PATCH 6/8] log event --- lambdas/export_rds_snapshot_to_s3/main.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lambdas/export_rds_snapshot_to_s3/main.py b/lambdas/export_rds_snapshot_to_s3/main.py index 0f9fda09c..568c369be 100644 --- a/lambdas/export_rds_snapshot_to_s3/main.py +++ b/lambdas/export_rds_snapshot_to_s3/main.py @@ -10,6 +10,9 @@ def lambda_handler(event, context): + print("## EVENT") + print(event) + bucket_name = os.environ["BUCKET_NAME"] iam_role_arn = os.environ["IAM_ROLE_ARN"] kms_key_id = os.environ["KMS_KEY_ID"] From 3cf33e8fc5045d1c8bc70254d8d04dac04dbe274 Mon Sep 17 00:00:00 2001 From: timburke-hackit Date: Thu, 2 Nov 2023 11:21:49 +0000 Subject: [PATCH 7/8] resource arn value --- terraform/modules/rds-snapshot-to-s3/iam.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/modules/rds-snapshot-to-s3/iam.tf b/terraform/modules/rds-snapshot-to-s3/iam.tf index 7b83e0f30..c77db0b4c 100644 --- a/terraform/modules/rds-snapshot-to-s3/iam.tf +++ b/terraform/modules/rds-snapshot-to-s3/iam.tf @@ -103,7 +103,7 @@ data "aws_iam_policy_document" "rds_snapshot_to_s3_lambda" { ] effect = "Allow" resources = [ - aws_iam_role.rds_snapshot_to_s3_lambda_role + aws_iam_role.rds_snapshot_to_s3_lambda_role.arn ] } From 6bcccfb884097606f1eb7680de2bca3c4f97277c Mon Sep 17 00:00:00 2001 From: timburke-hackit Date: Thu, 2 Nov 2023 11:53:47 +0000 Subject: [PATCH 8/8] update kms sid --- terraform/modules/rds-snapshot-to-s3/iam.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/modules/rds-snapshot-to-s3/iam.tf b/terraform/modules/rds-snapshot-to-s3/iam.tf index c77db0b4c..98b1b0514 100644 --- a/terraform/modules/rds-snapshot-to-s3/iam.tf +++ b/terraform/modules/rds-snapshot-to-s3/iam.tf @@ -119,7 +119,7 @@ data "aws_iam_policy_document" "rds_snapshot_to_s3_lambda" { } statement { - sid = "AllowKMSDecrypt" + sid = "AllowKMSAccess" actions = [ "kms:Encrypt", "kms:GenerateDataKey",