Skip to content

Commit

Permalink
Feature/reencryption refactor (#150)
Browse files Browse the repository at this point in the history
* revert existing re-encryption

* Update logging

* more logging

* add kms key tag to backup resource

* fix tag

* add check on kms tag

* add re-encrypt absract function

* fix reference to backup tags

* update rds encrypt checks

* catch error for no snapshots

* add param for reencrypt to sam template

* add func to check snaps in encrypt process

* fix error handling for no snapshot

* fixes

* more logging

* logging

* Fix syntax error in snapshot identifier

* remove debug logging

* readd docdb fix
  • Loading branch information
tarunmenon95 authored Jul 11, 2024
1 parent 9c813a2 commit 1c96810
Showing 13 changed files with 114 additions and 120 deletions.
1 change: 1 addition & 0 deletions Jenkinsfile
Original file line number Diff line number Diff line change
@@ -31,6 +31,7 @@ pipeline {
}

stage('Unit Tests') {
when { changeRequest target: 'master' }
steps {
script {
//Source Account
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
@@ -184,8 +184,8 @@ IAM role that Lambda is running under.

## Runtime environment

Shelvery requires Python3.6 to run. You can run it either from any server or local machine capable of interpreting
Python3.6 code, or as Amazon Lambda functions. All Shelvery code is written in such way that it supports
Shelvery requires Python3.11 to run. You can run it either from any server or local machine capable of interpreting
Python3.11 code, or as Amazon Lambda functions. All Shelvery code is written in such way that it supports
both CLI and Lambda execution.

## Backup lifecycle and retention periods
2 changes: 1 addition & 1 deletion deploy-sam-template.sh
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#!/bin/bash
set -e

SHELVERY_VERSION=0.9.9
SHELVERY_VERSION=0.9.10

# set DOCKERUSERID to current user. could be changed with -u uid
DOCKERUSERID="-u $(id -u)"
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from setuptools import setup

setup(name='shelvery', version='0.9.9', author='Base2Services R&D',
setup(name='shelvery', version='0.9.10', author='Base2Services R&D',
author_email='[email protected]',
url='http://github.com/base2Services/shelvery-aws-backups',
classifiers=[
2 changes: 1 addition & 1 deletion shelvery/__init__.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
__version__ = '0.9.9'
__version__ = '0.9.10'
LAMBDA_WAIT_ITERATION = 'lambda_wait_iteration'
S3_DATA_PREFIX = 'backups'
SHELVERY_DO_BACKUP_TAGS = ['True', 'true', '1', 'TRUE']
3 changes: 3 additions & 0 deletions shelvery/documentdb_backup.py
Original file line number Diff line number Diff line change
@@ -121,6 +121,9 @@ def copy_backup_to_region(self, backup_id: str, region: str) -> str:
CopyTags=False
)
return backup_id

def create_encrypted_backup(self, backup_id: str, kms_key: str, region: str) -> str:
return backup_id

def copy_shared_backup(self, source_account: str, source_backup: BackupResource):
docdb_client = AwsHelper.boto3_client('docdb', arn=self.role_arn, external_id=self.role_external_id)
4 changes: 4 additions & 0 deletions shelvery/ebs_backup.py
Original file line number Diff line number Diff line change
@@ -125,6 +125,10 @@ def copy_shared_backup(self, source_account: str, source_backup: BackupResource)
SourceRegion=source_backup.region
)
return snap['SnapshotId']

def create_encrypted_backup(self, backup_id: str, kms_key: str, region: str) -> str:
return backup_id

# collect all volumes tagged with given tag, in paginated manner
def collect_volumes(self, tag_name: str):
load_volumes = True
3 changes: 3 additions & 0 deletions shelvery/ec2ami_backup.py
Original file line number Diff line number Diff line change
@@ -221,3 +221,6 @@ def share_backup_with_account(self, backup_region: str, backup_id: str, aws_acco
},
UserIds=[aws_account_id],
OperationType='add')

def create_encrypted_backup(self, backup_id: str, kms_key: str, region: str) -> str:
return backup_id
24 changes: 19 additions & 5 deletions shelvery/engine.py
Original file line number Diff line number Diff line change
@@ -256,6 +256,12 @@ def create_backups(self) -> List[BackupResource]:

dr_regions = RuntimeConfig.get_dr_regions(backup_resource.entity_resource.tags, self)
backup_resource.tags[f"{RuntimeConfig.get_tag_prefix()}:dr_regions"] = ','.join(dr_regions)

re_encrypt_key = RuntimeConfig.get_reencrypt_kms_key_id(backup_resource.entity_resource.tags, self)
if re_encrypt_key := RuntimeConfig.get_reencrypt_kms_key_id(backup_resource.entity_resource.tags, self):
backup_resource.tags[f"{RuntimeConfig.get_tag_prefix()}:config:shelvery_reencrypt_kms_key_id"] = re_encrypt_key


self.logger.info(f"Processing {resource_type} with id {r.resource_id}")
self.logger.info(f"Creating backup {backup_resource.name}")

@@ -661,6 +667,11 @@ def do_share_backup(self, map_args={}, **kwargs):
backup_region = kwargs['Region']
destination_account_id = kwargs['AwsAccountId']
backup_resource = self.get_backup_resource(backup_region, backup_id)

if re_encrypt_key := RuntimeConfig.get_reencrypt_kms_key_id(backup_resource.tags, self):
self.logger.info(f"KMS Key detected during share for {backup_resource.backup_id}")
backup_id = self.create_encrypted_backup(backup_id, re_encrypt_key, backup_region)

# if backup is not available, exit and rely on recursive lambda call do share backup
# in non lambda mode this should never happen
if RuntimeConfig.is_offload_queueing(self):
@@ -675,11 +686,8 @@ def do_share_backup(self, map_args={}, **kwargs):

self.logger.info(f"Do share backup {backup_id} ({backup_region}) with {destination_account_id}")
try:
new_backup_id = self.share_backup_with_account(backup_region, backup_id, destination_account_id)
#assign new backup id if new snapshot is created (eg: re-encrypted rds snapshot)
backup_id = new_backup_id if new_backup_id else backup_id
self.logger.info(f"Shared backup {backup_id} ({backup_region}) with {destination_account_id}")
backup_resource = self.get_backup_resource(backup_region, backup_id)
self.share_backup_with_account(backup_region, backup_id, destination_account_id)
backup_resource = self.get_backup_resource(backup_region, backup_id)
self._write_backup_data(
backup_resource,
self._get_data_bucket(backup_region),
@@ -840,3 +848,9 @@ def get_backup_resource(self, backup_region: str, backup_id: str) -> BackupResou
"""
Get Backup Resource within region, identified by its backup_id
"""

@abstractmethod
def create_encrypted_backup(self, backup_id: str, kms_key: str, backup_region: str) -> str:
"""
Re-encrypt an existing backup with a new KMS key, returns the new backup id
"""
87 changes: 34 additions & 53 deletions shelvery/rds_backup.py
Original file line number Diff line number Diff line change
@@ -95,76 +95,57 @@ def get_existing_backups(self, backup_tag_prefix: str) -> List[BackupResource]:

def share_backup_with_account(self, backup_region: str, backup_id: str, aws_account_id: str):
rds_client = AwsHelper.boto3_client('rds', region_name=backup_region, arn=self.role_arn, external_id=self.role_external_id)
backup_resource = self.get_backup_resource(backup_region, backup_id)
kms_key = RuntimeConfig.get_reencrypt_kms_key_id(backup_resource.tags, self)

# if a re-encrypt key is provided, create new re-encrypted snapshot and share that instead
if kms_key:
self.logger.info(f"Re-encrypt KMS Key found, creating new backup with {kms_key}")
# create re-encrypted backup
backup_id = self.copy_backup_to_region(backup_id, backup_region)
self.logger.info(f"Creating new encrypted backup {backup_id}")
# wait till new snapshot is available
if not self.wait_backup_available(backup_region=backup_region,
backup_id=backup_id,
lambda_method='do_share_backup',
lambda_args={}):
return
self.logger.info(f"New encrypted backup {backup_id} created")

#Get new snapshot ARN
snapshots = rds_client.describe_db_snapshots(DBSnapshotIdentifier=backup_id)
snapshot_arn = snapshots['DBSnapshots'][0]['DBSnapshotArn']

#Update tags with '-re-encrypted' suffix
self.logger.info(f"Updating tags for new snapshot - {backup_id}")
tags = self.get_backup_resource(backup_region, backup_id).tags
tags.update({'Name': backup_id, 'shelvery:name': backup_id})
tag_list = [{'Key': key, 'Value': value} for key, value in tags.items()]
rds_client.add_tags_to_resource(
ResourceName=snapshot_arn,
Tags=tag_list
)
created_new_encrypted_snapshot = True
else:
self.logger.info(f"No re-encrypt key detected")
created_new_encrypted_snapshot = False

rds_client.modify_db_snapshot_attribute(
DBSnapshotIdentifier=backup_id,
AttributeName='restore',
ValuesToAdd=[aws_account_id]
)
# if re-encryption occured, clean up old snapshot
if created_new_encrypted_snapshot:
# delete old snapshot
self.delete_backup(backup_resource)
self.logger.info(f"Cleaning up un-encrypted backup: {backup_resource.backup_id}")

return backup_id

def copy_backup_to_region(self, backup_id: str, region: str) -> str:
local_region = boto3.session.Session().region_name
client_local = AwsHelper.boto3_client('rds', arn=self.role_arn, external_id=self.role_external_id)
rds_client = AwsHelper.boto3_client('rds', region_name=region, arn=self.role_arn, external_id=self.role_external_id)
snapshots = client_local.describe_db_snapshots(DBSnapshotIdentifier=backup_id)
snapshot = snapshots['DBSnapshots'][0]
backup_resource = self.get_backup_resource(local_region, backup_id)
kms_key = RuntimeConfig.get_reencrypt_kms_key_id(backup_resource.tags, self)
rds_client.copy_db_snapshot(
SourceDBSnapshotIdentifier=snapshot['DBSnapshotArn'],
TargetDBSnapshotIdentifier=backup_id,
SourceRegion=local_region,
# tags are created explicitly
CopyTags=False
)
return backup_id

def snapshot_exists(self, client, backup_id):
try:
response = client.describe_db_snapshots(DBSnapshotIdentifier=backup_id)
snapshots = response.get('DBSnapshots', [])
return bool(snapshots)
except ClientError as e:
if e.response['Error']['Code'] == 'DBSnapshotNotFound':
return False
else:
print(e.response['Error']['Code'])
raise e

def create_encrypted_backup(self, backup_id: str, kms_key: str, region: str) -> str:
local_region = boto3.session.Session().region_name
client_local = AwsHelper.boto3_client('rds', arn=self.role_arn, external_id=self.role_external_id)
rds_client = AwsHelper.boto3_client('rds', region_name=region, arn=self.role_arn, external_id=self.role_external_id)
snapshots = client_local.describe_db_snapshots(DBSnapshotIdentifier=backup_id)
snapshot = snapshots['DBSnapshots'][0]
backup_id = f'{backup_id}-re-encrypted'

if self.snapshot_exists(rds_client, backup_id):
return backup_id

rds_client_params = {
'SourceDBSnapshotIdentifier': snapshot['DBSnapshotArn'],
'TargetDBSnapshotIdentifier': backup_id,
'SourceRegion': local_region,
# tags are created explicitly
'CopyTags': False
'CopyTags': True,
'KmsKeyId': kms_key,
}
# add kms key params if reencrypt key is defined
if kms_key is not None:
backup_id = f'{backup_id}-re-encrypted'
rds_client_params['KmsKeyId'] = kms_key
rds_client_params['CopyTags'] = True
rds_client_params['TargetDBSnapshotIdentifier'] = backup_id

rds_client.copy_db_snapshot(**rds_client_params)
return backup_id

90 changes: 35 additions & 55 deletions shelvery/rds_cluster_backup.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
from tracemalloc import Snapshot
import boto3

from shelvery.runtime_config import RuntimeConfig
@@ -97,74 +96,57 @@ def get_existing_backups(self, backup_tag_prefix: str) -> List[BackupResource]:

def share_backup_with_account(self, backup_region: str, backup_id: str, aws_account_id: str):
rds_client = AwsHelper.boto3_client('rds', region_name=backup_region, arn=self.role_arn, external_id=self.role_external_id)
backup_resource = self.get_backup_resource(backup_region, backup_id)
kms_key = RuntimeConfig.get_reencrypt_kms_key_id(backup_resource.tags, self)

# if a re-encrypt key is provided, create new re-encrypted snapshot and share that instead
if kms_key:
self.logger.info(f"Re-encrypt KMS Key found, creating new backup with {kms_key}")
# create re-encrypted backup
backup_id = self.copy_backup_to_region(backup_id, backup_region)
self.logger.info(f"Creating new encrypted backup {backup_id}")
# wait till new snapshot is available
if not self.wait_backup_available(backup_region=backup_region,
backup_id=backup_id,
lambda_method='do_share_backup',
lambda_args={}):
return
self.logger.info(f"New encrypted backup {backup_id} created")

#Get new snapshot ARN
snapshots = rds_client.describe_db_cluster_snapshots(DBClusterSnapshotIdentifier=backup_id)
snapshot_arn = snapshots['DBClusterSnapshots'][0]['DBClusterSnapshotArn']

#Update tags with '-re-encrypted' suffix
self.logger.info(f"Updating tags for new snapshot - {backup_id}")
tags = self.get_backup_resource(backup_region, backup_id).tags
tags.update({'Name': backup_id, 'shelvery:name': backup_id})
tag_list = [{'Key': key, 'Value': value} for key, value in tags.items()]
rds_client.add_tags_to_resource(
ResourceName=snapshot_arn,
Tags=tag_list
)
created_new_encrypted_snapshot = True
else:
self.logger.info(f"No re-encrypt key detected")
created_new_encrypted_snapshot = False

rds_client.modify_db_cluster_snapshot_attribute(
DBClusterSnapshotIdentifier=backup_id,
AttributeName='restore',
ValuesToAdd=[aws_account_id]
)
# if re-encryption occured, clean up old snapshot
if created_new_encrypted_snapshot:
# delete old snapshot
self.delete_backup(backup_resource)
self.logger.info(f"Cleaning up un-encrypted backup: {backup_resource.backup_id}")

return backup_id

def copy_backup_to_region(self, backup_id: str, region: str) -> str:
local_region = boto3.session.Session().region_name
client_local = AwsHelper.boto3_client('rds', arn=self.role_arn, external_id=self.role_external_id)
rds_client = AwsHelper.boto3_client('rds', region_name=region)
snapshots = client_local.describe_db_cluster_snapshots(DBClusterSnapshotIdentifier=backup_id)
snapshot = snapshots['DBClusterSnapshots'][0]
backup_resource = self.get_backup_resource(local_region, backup_id)
kms_key = RuntimeConfig.get_reencrypt_kms_key_id(backup_resource.tags, self)
rds_client.copy_db_cluster_snapshot(
SourceDBClusterSnapshotIdentifier=snapshot['DBClusterSnapshotArn'],
TargetDBClusterSnapshotIdentifier=backup_id,
SourceRegion=local_region,
# tags are created explicitly
CopyTags=False
)
return backup_id

def snapshot_exists(client, backup_id):
try:
response = client.describe_db_cluster_snapshots(DBClusterSnapshotIdentifier=backup_id)
snapshots = response.get('DBClusterSnapshots', [])
return bool(snapshots)
except ClientError as e:
if e.response['Error']['Code'] == 'DBClusterSnapshotNotFound':
return False
else:
print(e.response['Error']['Code'])
raise e

def create_encrypted_backup(self, backup_id: str, kms_key: str, region: str) -> str:
local_region = boto3.session.Session().region_name
client_local = AwsHelper.boto3_client('rds', arn=self.role_arn, external_id=self.role_external_id)
rds_client = AwsHelper.boto3_client('rds', region_name=region)
snapshots = client_local.describe_db_cluster_snapshots(DBClusterSnapshotIdentifier=backup_id)
snapshot = snapshots['DBClusterSnapshots'][0]
backup_id = f'{backup_id}-re-encrypted'

if self.snapshot_exists(rds_client, backup_id):
return backup_id

rds_client_params = {
'SourceDBClusterSnapshotIdentifier': snapshot['DBClusterSnapshotArn'],
'TargetDBClusterSnapshotIdentifier': backup_id,
'SourceRegion': local_region,
'CopyTags': False
'CopyTags': True,
'KmsKeyId': kms_key,
}
# add kms key params if re-encrypt key is defined
if kms_key is not None:
backup_id = f'{backup_id}-re-encrypted'
rds_client_params['KmsKeyId'] = kms_key
rds_client_params['CopyTags'] = True
rds_client_params['TargetDBClusterSnapshotIdentifier'] = backup_id

rds_client.copy_db_cluster_snapshot(**rds_client_params)
return backup_id
@@ -255,7 +237,6 @@ def get_all_clusters(self, rds_client):
db_clusters = []
# temporary list of api models, as calls are batched
temp_clusters = rds_client.describe_db_clusters()

db_clusters.extend(temp_clusters['DBClusters'])
# collect database instances
while 'Marker' in temp_clusters:
@@ -304,9 +285,8 @@ def collect_all_snapshots(self, rds_client):
self.logger.info(f"Collected {len(tmp_snapshots['DBClusterSnapshots'])} manual snapshots. Continuing collection...")
tmp_snapshots = rds_client.describe_db_cluster_snapshots(SnapshotType='manual', Marker=tmp_snapshots['Marker'])
all_snapshots.extend(tmp_snapshots['DBClusterSnapshots'])

all_snapshots = [snapshot for snapshot in all_snapshots if snapshot.get('Engine') != 'docdb']

all_snapshots = [snapshot for snapshot in all_snapshots if snapshot.get('Engine') != 'docdb']
self.logger.info(f"Collected {len(all_snapshots)} manual snapshots.")
self.populate_snap_entity_resource(all_snapshots)

3 changes: 3 additions & 0 deletions shelvery/redshift_backup.py
Original file line number Diff line number Diff line change
@@ -215,6 +215,9 @@ def copy_backup_to_region(self, backup_id: str, region: str) -> str:
"using EnableSnapshotCopy API Call.")
pass

def create_encrypted_backup(self, backup_id: str, kms_key: str, region: str) -> str:
return backup_id

def is_backup_available(self, backup_region: str, backup_id: str) -> bool:
"""
Determine whether backup has completed and is available to be copied
Loading

0 comments on commit 1c96810

Please sign in to comment.