snapshot reencryption (#142)

shelvery/engine.py

@@ -675,7 +675,10 @@ def do_share_backup(self, map_args={}, **kwargs):
 
         self.logger.info(f"Do share backup {backup_id} ({backup_region}) with {destination_account_id}")
         try:
-            self.share_backup_with_account(backup_region, backup_id, destination_account_id)
+            new_backup_id = self.share_backup_with_account(backup_region, backup_id, destination_account_id)
+            #assign new backup id if new snapshot is created (eg: re-encrypted rds snapshot)
+            backup_id = new_backup_id if new_backup_id else backup_id
+            self.logger.info(f"Shared backup {backup_id} ({backup_region}) with {destination_account_id}")
             backup_resource = self.get_backup_resource(backup_region, backup_id)
             self._write_backup_data(
                 backup_resource,

shelvery/rds_backup.py

@@ -95,25 +95,77 @@ def get_existing_backups(self, backup_tag_prefix: str) -> List[BackupResource]:
 
     def share_backup_with_account(self, backup_region: str, backup_id: str, aws_account_id: str):
         rds_client = AwsHelper.boto3_client('rds', region_name=backup_region, arn=self.role_arn, external_id=self.role_external_id)
+        backup_resource = self.get_backup_resource(backup_region, backup_id)
+        kms_key = RuntimeConfig.get_reencrypt_kms_key_id(backup_resource.tags, self)
+
+        # if a re-encrypt key is provided, create new re-encrypted snapshot and share that instead
+        if kms_key:
+            self.logger.info(f"Re-encrypt KMS Key found, creating new backup with {kms_key}")
+            # create re-encrypted backup
+            backup_id = self.copy_backup_to_region(backup_id, backup_region)
+            self.logger.info(f"Creating new encrypted backup {backup_id}")
+            # wait till new snapshot is available
+            if not self.wait_backup_available(backup_region=backup_region,
+                backup_id=backup_id,
+                lambda_method='do_share_backup',
+                lambda_args={}):
+                return
+            self.logger.info(f"New encrypted backup {backup_id} created")
+
+            #Get new snapshot ARN
+            snapshots = rds_client.describe_db_snapshots(DBSnapshotIdentifier=backup_id)
+            snapshot_arn = snapshots['DBSnapshots'][0]['DBSnapshotArn']
+
+            #Update tags with '-re-encrypted' suffix
+            self.logger.info(f"Updating tags for new snapshot - {backup_id}")
+            tags = self.get_backup_resource(backup_region, backup_id).tags
+            tags.update({'Name': backup_id, 'shelvery:name': backup_id})
+            tag_list = [{'Key': key, 'Value': value} for key, value in tags.items()]
+            rds_client.add_tags_to_resource(
+                ResourceName=snapshot_arn,
+                Tags=tag_list
+            )
+            created_new_encrypted_snapshot = True
+        else:
+            self.logger.info(f"No re-encrypt key detected")
+            created_new_encrypted_snapshot = False 
+
         rds_client.modify_db_snapshot_attribute(
             DBSnapshotIdentifier=backup_id,
             AttributeName='restore',
             ValuesToAdd=[aws_account_id]
         )
+        # if re-encryption occured, clean up old snapshot
+        if created_new_encrypted_snapshot:
+            # delete old snapshot
+            self.delete_backup(backup_resource)
+            self.logger.info(f"Cleaning up un-encrypted backup: {backup_resource.backup_id}")
+
+        return backup_id
 
     def copy_backup_to_region(self, backup_id: str, region: str) -> str:
         local_region = boto3.session.Session().region_name
         client_local = AwsHelper.boto3_client('rds', arn=self.role_arn, external_id=self.role_external_id)
         rds_client = AwsHelper.boto3_client('rds', region_name=region, arn=self.role_arn, external_id=self.role_external_id)
         snapshots = client_local.describe_db_snapshots(DBSnapshotIdentifier=backup_id)
         snapshot = snapshots['DBSnapshots'][0]
-        rds_client.copy_db_snapshot(
-            SourceDBSnapshotIdentifier=snapshot['DBSnapshotArn'],
-            TargetDBSnapshotIdentifier=backup_id,
-            SourceRegion=local_region,
+        backup_resource = self.get_backup_resource(local_region, backup_id)
+        kms_key = RuntimeConfig.get_reencrypt_kms_key_id(backup_resource.tags, self)
+        rds_client_params = {
+            'SourceDBSnapshotIdentifier': snapshot['DBSnapshotArn'],
+            'TargetDBSnapshotIdentifier': backup_id,
+            'SourceRegion': local_region,
             # tags are created explicitly
-            CopyTags=False
-        )
+            'CopyTags': False
+        }
+         # add kms key params if reencrypt key is defined
+        if kms_key is not None:
+            backup_id = f'{backup_id}-re-encrypted'
+            rds_client_params['KmsKeyId'] = kms_key
+            rds_client_params['CopyTags'] = True
+            rds_client_params['TargetDBSnapshotIdentifier'] = backup_id
+
+        rds_client.copy_db_snapshot(**rds_client_params)
         return backup_id
 
     def get_backup_resource(self, backup_region: str, backup_id: str) -> BackupResource:

shelvery/rds_cluster_backup.py

@@ -96,25 +96,76 @@ def get_existing_backups(self, backup_tag_prefix: str) -> List[BackupResource]:
 
     def share_backup_with_account(self, backup_region: str, backup_id: str, aws_account_id: str):
         rds_client = AwsHelper.boto3_client('rds', region_name=backup_region, arn=self.role_arn, external_id=self.role_external_id)
+        backup_resource = self.get_backup_resource(backup_region, backup_id)
+        kms_key = RuntimeConfig.get_reencrypt_kms_key_id(backup_resource.tags, self)
+
+        # if a re-encrypt key is provided, create new re-encrypted snapshot and share that instead
+        if kms_key:
+            self.logger.info(f"Re-encrypt KMS Key found, creating new backup with {kms_key}")
+            # create re-encrypted backup
+            backup_id = self.copy_backup_to_region(backup_id, backup_region)
+            self.logger.info(f"Creating new encrypted backup {backup_id}")
+            # wait till new snapshot is available
+            if not self.wait_backup_available(backup_region=backup_region,
+                backup_id=backup_id,
+                lambda_method='do_share_backup',
+                lambda_args={}):
+                return
+            self.logger.info(f"New encrypted backup {backup_id} created")
+
+            #Get new snapshot ARN 
+            snapshots = rds_client.describe_db_cluster_snapshots(DBClusterSnapshotIdentifier=backup_id)
+            snapshot_arn = snapshots['DBClusterSnapshots'][0]['DBClusterSnapshotArn']
+
+            #Update tags with '-re-encrypted' suffix
+            self.logger.info(f"Updating tags for new snapshot - {backup_id}")
+            tags = self.get_backup_resource(backup_region, backup_id).tags
+            tags.update({'Name': backup_id, 'shelvery:name': backup_id})
+            tag_list = [{'Key': key, 'Value': value} for key, value in tags.items()]
+            rds_client.add_tags_to_resource(
+                ResourceName=snapshot_arn,
+                Tags=tag_list
+            )
+            created_new_encrypted_snapshot = True
+        else:
+            self.logger.info(f"No re-encrypt key detected")
+            created_new_encrypted_snapshot = False 
+
         rds_client.modify_db_cluster_snapshot_attribute(
             DBClusterSnapshotIdentifier=backup_id,
             AttributeName='restore',
             ValuesToAdd=[aws_account_id]
         )
+        # if re-encryption occured, clean up old snapshot
+        if created_new_encrypted_snapshot:
+            # delete old snapshot
+            self.delete_backup(backup_resource)
+            self.logger.info(f"Cleaning up un-encrypted backup: {backup_resource.backup_id}")
+
+        return backup_id
 
     def copy_backup_to_region(self, backup_id: str, region: str) -> str:
         local_region = boto3.session.Session().region_name
         client_local = AwsHelper.boto3_client('rds', arn=self.role_arn, external_id=self.role_external_id)
         rds_client = AwsHelper.boto3_client('rds', region_name=region)
         snapshots = client_local.describe_db_cluster_snapshots(DBClusterSnapshotIdentifier=backup_id)
         snapshot = snapshots['DBClusterSnapshots'][0]
-        rds_client.copy_db_cluster_snapshot(
-            SourceDBClusterSnapshotIdentifier=snapshot['DBClusterSnapshotArn'],
-            TargetDBClusterSnapshotIdentifier=backup_id,
-            SourceRegion=local_region,
-            # tags are created explicitly
-            CopyTags=False
-        )
+        backup_resource = self.get_backup_resource(local_region, backup_id)
+        kms_key = RuntimeConfig.get_reencrypt_kms_key_id(backup_resource.tags, self)
+        rds_client_params = {
+            'SourceDBClusterSnapshotIdentifier': snapshot['DBClusterSnapshotArn'],
+            'TargetDBClusterSnapshotIdentifier': backup_id,
+            'SourceRegion': local_region,
+            'CopyTags': False
+        }
+        # add kms key params if re-encrypt key is defined
+        if kms_key is not None:
+            backup_id = f'{backup_id}-re-encrypted'
+            rds_client_params['KmsKeyId'] = kms_key
+            rds_client_params['CopyTags'] = True
+            rds_client_params['TargetDBClusterSnapshotIdentifier'] = backup_id
+
+        rds_client.copy_db_cluster_snapshot(**rds_client_params)
         return backup_id
 
     def copy_shared_backup(self, source_account: str, source_backup: BackupResource):

shelvery/runtime_config.py

@@ -62,6 +62,7 @@ class RuntimeConfig:
                             when enabled 'shelvery_copy_kms_key_id' must also be set.
     shelvery_copy_kms_key_id - when copying a shared snapshot, you can specify a different kms key used to encrypt the original snapshot.
                                Note that when copying to a new key, the shelvery requires access to both the new key and the original key.
+    shelvery_reencrypt_kms_key_id - when re-encrypting a snapshot with a new KMS key before sharing it to a new account.               
     """
 
     DEFAULT_KEEP_DAILY = 14
@@ -102,7 +103,8 @@ class RuntimeConfig:
         'shelvery_sqs_queue_wait_period': 0,
         'shelvery_ignore_invalid_resource_state': False,
         'shelvery_encrypt_copy': False,
-        'shelvery_copy_kms_key_id': None
+        'shelvery_copy_kms_key_id': None,
+        'shelvery_reencrypt_kms_key_id': None
     }
 
     @classmethod
@@ -334,4 +336,8 @@ def get_encrypt_copy(cls, resource_tags, engine):
 
     @classmethod
     def get_copy_kms_key_id(cls, resource_tags, engine):
-        return cls.get_conf_value('shelvery_copy_kms_key_id', resource_tags, engine.lambda_payload)
+        return cls.get_conf_value('shelvery_copy_kms_key_id', resource_tags, engine.lambda_payload)
+
+    @classmethod
+    def get_reencrypt_kms_key_id(cls, resource_tags, engine):
+        return cls.get_conf_value('shelvery_reencrypt_kms_key_id', resource_tags, engine.lambda_payload)

shelvery_tests/rds_cluster_integration_test.py

@@ -50,7 +50,7 @@ def setUp(self):
 
        # Complete initial setup and create service client
         initSetup(self,'rds')
-        rdsclient = AwsHelper.boto3_client('docdb', region_name='ap-southeast-2')
+        rdsclient = AwsHelper.boto3_client('rds', region_name='ap-southeast-2')
 
 
         #Get cluster name