diff --git a/lambdas/export_rds_snapshot_to_s3/main.py b/lambdas/export_rds_snapshot_to_s3/main.py index 50cedd42c..568c369be 100644 --- a/lambdas/export_rds_snapshot_to_s3/main.py +++ b/lambdas/export_rds_snapshot_to_s3/main.py @@ -1,5 +1,6 @@ import logging import os +import ast import boto3 from botocore.exceptions import ClientError @@ -9,12 +10,17 @@ def lambda_handler(event, context): - snapshot_identifier = event["detail"]["SnapshotIdentifier"] - source_arn = event["detail"]["SourceArn"] + print("## EVENT") + print(event) + bucket_name = os.environ["BUCKET_NAME"] iam_role_arn = os.environ["IAM_ROLE_ARN"] kms_key_id = os.environ["KMS_KEY_ID"] + event = ast.literal_eval(event) + snapshot_identifier = event["detail"]["SourceIdentifier"] + source_arn = event["detail"]["SourceArn"] + try: rds.start_export_task( ExportTaskIdentifier=snapshot_identifier, diff --git a/lambdas/rds_snapshot_export_s3_to_s3_copier/main.py b/lambdas/rds_snapshot_export_s3_to_s3_copier/main.py index 6375a17b4..f62067a1c 100644 --- a/lambdas/rds_snapshot_export_s3_to_s3_copier/main.py +++ b/lambdas/rds_snapshot_export_s3_to_s3_copier/main.py @@ -1,4 +1,5 @@ import os +import ast import boto3 @@ -57,8 +58,8 @@ def s3_copy_folder( continue source_key_split = source_key.split("/") parquet_file_name = source_key_split[-1] - database_name = source_key_split("/")[1] - table_name = source_key_split("/")[2] + database_name = source_key_split[1] + table_name = source_key_split[2] copy_object_params = { "Bucket": target_bucket, "CopySource": f"{source_bucket}/{source_key}", @@ -85,6 +86,8 @@ def start_workflow_run(workflow_name: str, glue_client): def lambda_handler(event, context) -> None: + print("## EVENT") + print(event) s3 = boto3.client("s3") source_bucket = os.environ["SOURCE_BUCKET"] @@ -98,7 +101,8 @@ def lambda_handler(event, context) -> None: else: target_prefix = "" - snapshot_id = event["detail"]["SnapshotIdentifier"] + event = ast.literal_eval(event) + snapshot_id = event["detail"]["SourceIdentifier"] s3_copy_folder( s3, source_bucket, source_prefix, target_bucket, target_prefix, snapshot_id diff --git a/terraform/modules/rds-snapshot-to-s3/iam.tf b/terraform/modules/rds-snapshot-to-s3/iam.tf index 96028cee5..98b1b0514 100644 --- a/terraform/modules/rds-snapshot-to-s3/iam.tf +++ b/terraform/modules/rds-snapshot-to-s3/iam.tf @@ -64,6 +64,18 @@ data "aws_iam_policy_document" "lambda_assume_role" { type = "Service" } } + + statement { + actions = [ + "sts:AssumeRole" + ] + principals { + identifiers = [ + "export.rds.amazonaws.com" + ] + type = "Service" + } + } } # RDS Snapshot to S3 lambda IAM @@ -86,9 +98,13 @@ data "aws_iam_policy_document" "rds_snapshot_to_s3_lambda" { } statement { - actions = ["iam:PassRole"] - effect = "Allow" - resources = [var.rds_snapshot_service_arn] + actions = [ + "iam:PassRole" + ] + effect = "Allow" + resources = [ + aws_iam_role.rds_snapshot_to_s3_lambda_role.arn + ] } statement { @@ -103,10 +119,17 @@ data "aws_iam_policy_document" "rds_snapshot_to_s3_lambda" { } statement { - sid = "AllowKMSDecrypt" + sid = "AllowKMSAccess" actions = [ + "kms:Encrypt", + "kms:GenerateDataKey", "kms:Decrypt", - "kms:GenerateDataKey*" + "kms:GenerateDataKeyPairWithoutPlaintext", + "kms:ReEncryptFrom", + "kms:ReEncryptTo", + "kms:CreateGrant", + "kms:DescribeKey", + "kms:RetireGrant" ] effect = "Allow" resources = [